Historia cen na prawdziwych danych: ogloszenia, NBP i GUS
Deploy Polska Lokalnie / deploy (push) Successful in 12s
Deploy Polska Lokalnie / deploy (push) Successful in 12s
Strona /historia-cen pokazywala pusty wykres (wczesniej zmyslony). Teraz:
Ceny ofertowe z ogloszen:
- Tabela price_snapshots: codzienny zapis srednich cen za m2 z opublikowanych
ogloszen per miasto i dzielnica, typ (mieszkanie/dom) i rynek. Zapis co noc
(02:30) i przy starcie, gdy brakuje dzisiejszego. Historii sprzed pierwszego
zapisu nie odtwarzamy.
Ceny transakcyjne ze zrodel publicznych (porownanie, cale miasto, mieszkania):
- NBP: plik ceny_mieszkan.xlsx (baza BaRN) - 17 miast, rynek pierwotny
i wtorny, od III kw. 2006. Odczyt XLSX bez nowych zaleznosci.
- GUS: API Banku Danych Lokalnych (temat P3794) - miasta na prawach powiatu,
kwartalnie od 2010, z opoznieniem ok. roku.
- Import raz w tygodniu i przy starcie, gdy dane sa starsze niz tydzien;
blad zrodla nie kasuje poprzednich danych.
- GET /api/price-history (publiczny): historia, biezaca srednia z ogloszen,
od kiedy zbieramy dane oraz serie NBP i GUS.
Frontend:
- Okresy "Ostatnie 5 lat" (domyslny), "Od 2006 (NBP)", "2026 do dzis";
trzy serie z legenda, dymek ze wszystkimi zrodlami, statystyki i wnioski
z prawdziwych liczb, porownanie cen ofertowych z NBP i GUS oraz roznica
i srednia, gdy zrodla sie roznia. Ceny w "Porownaj lokalizacje".
- Poprawione dopasowanie lokalizacji ("Warszawa" otwierala "Warszawa,
Mokotów"), usuniete resztki zmyslonych cen i opcja "Lokal uzytkowy".
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -49,6 +49,8 @@ public class SecurityConfig {
|
||||
.requestMatchers(HttpMethod.GET, "/api/listings", "/api/listings/**").permitAll()
|
||||
// Podglad ogloszenia dla crawlerow social media (Open Graph) - publiczny, tylko GET.
|
||||
.requestMatchers(HttpMethod.GET, "/api/og/**").permitAll()
|
||||
// Historia srednich cen z ogloszen - publiczna, jak same ogloszenia.
|
||||
.requestMatchers(HttpMethod.GET, "/api/price-history").permitAll()
|
||||
.requestMatchers(HttpMethod.GET, "/api/articles", "/api/articles/**").permitAll()
|
||||
.requestMatchers(HttpMethod.POST, "/api/articles").authenticated()
|
||||
.requestMatchers(HttpMethod.GET, "/api/banners").permitAll()
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
package pl.polskalokalnie.pricehistory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.scheduling.annotation.Async;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
import pl.polskalokalnie.listing.ListingRepository;
|
||||
import pl.polskalokalnie.listing.ListingStatus;
|
||||
import pl.polskalokalnie.listing.PropertyListing;
|
||||
import pl.polskalokalnie.pricehistory.ExternalQuarterPrice.Source;
|
||||
|
||||
/**
|
||||
* Import cen transakcyjnych z NBP (plik XLSX) i GUS (API BDL) - zrodla porownawcze dla historii cen.
|
||||
* Raz w tygodniu oraz przy starcie, gdy dane sa starsze niz tydzien. Blad zrodla nie rusza starych danych.
|
||||
*/
|
||||
@Service
|
||||
public class ExternalPriceImportService {
|
||||
|
||||
static final String NBP_URL = "https://static.nbp.pl/dane/rynek-nieruchomosci/ceny_mieszkan.xlsx";
|
||||
private static final long MAX_DOWNLOAD_BYTES = 20L * 1024 * 1024;
|
||||
private static final int GUS_FROM_YEAR = 2010;
|
||||
private static final Logger log = LoggerFactory.getLogger(ExternalPriceImportService.class);
|
||||
|
||||
private final ExternalQuarterPriceRepository repository;
|
||||
private final ListingRepository listingRepository;
|
||||
private final GusBdlClient gusClient;
|
||||
private final TransactionTemplate transaction;
|
||||
private final HttpClient http = HttpClient.newBuilder()
|
||||
.connectTimeout(Duration.ofSeconds(15))
|
||||
.followRedirects(HttpClient.Redirect.NORMAL)
|
||||
.build();
|
||||
|
||||
public ExternalPriceImportService(ExternalQuarterPriceRepository repository, ListingRepository listingRepository,
|
||||
GusBdlClient gusClient, TransactionTemplate transaction) {
|
||||
this.repository = repository;
|
||||
this.listingRepository = listingRepository;
|
||||
this.gusClient = gusClient;
|
||||
this.transaction = transaction;
|
||||
}
|
||||
|
||||
@Async
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void importOnStartupIfStale() {
|
||||
Instant weekAgo = Instant.now().minus(7, ChronoUnit.DAYS);
|
||||
if (repository.findLastImport(Source.NBP).map(last -> last.isBefore(weekAgo)).orElse(true)) {
|
||||
importNbp();
|
||||
}
|
||||
if (repository.findLastImport(Source.GUS).map(last -> last.isBefore(weekAgo)).orElse(true)) {
|
||||
importGus();
|
||||
}
|
||||
}
|
||||
|
||||
@Scheduled(cron = "0 10 4 * * MON", zone = "Europe/Warsaw")
|
||||
public void importWeekly() {
|
||||
importNbp();
|
||||
importGus();
|
||||
}
|
||||
|
||||
public int importNbp() {
|
||||
try {
|
||||
List<NbpXlsxParser.Row> rows;
|
||||
try (InputStream xlsx = download(NBP_URL)) {
|
||||
rows = NbpXlsxParser.parse(xlsx);
|
||||
}
|
||||
if (rows.isEmpty()) {
|
||||
log.warn("NBP: w pliku nie znaleziono cen transakcyjnych - zostawiam poprzednie dane");
|
||||
return 0;
|
||||
}
|
||||
Instant now = Instant.now();
|
||||
transaction.executeWithoutResult(status -> {
|
||||
repository.deleteBySource(Source.NBP);
|
||||
repository.saveAll(rows.stream()
|
||||
.map(row -> new ExternalQuarterPrice(Source.NBP, row.year(), row.quarter(), row.market(),
|
||||
row.city(), row.pricePerM2(), now))
|
||||
.toList());
|
||||
});
|
||||
log.info("NBP: zaimportowano {} cen kwartalnych", rows.size());
|
||||
return rows.size();
|
||||
} catch (IOException | RuntimeException ex) {
|
||||
log.warn("NBP: import nieudany ({}) - zostawiam poprzednie dane", ex.getMessage());
|
||||
return 0;
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** GUS dla miast z ogloszen i miast z danych NBP - tylko miasta na prawach powiatu maja wlasne dane. */
|
||||
public int importGus() {
|
||||
int imported = 0;
|
||||
try {
|
||||
List<GusBdlClient.Variable> variables = gusClient.variables();
|
||||
int toYear = PriceSnapshotService.today().getYear();
|
||||
for (String city : citiesToTrack()) {
|
||||
try {
|
||||
String unitId = gusClient.cityUnitId(city);
|
||||
if (unitId != null) {
|
||||
List<GusBdlClient.QuarterValue> values = gusClient.quarterlyPrices(unitId, variables, GUS_FROM_YEAR, toYear);
|
||||
if (!values.isEmpty()) {
|
||||
Instant now = Instant.now();
|
||||
transaction.executeWithoutResult(status -> {
|
||||
repository.deleteBySourceAndCityKey(Source.GUS, PriceSnapshotService.key(city));
|
||||
repository.saveAll(values.stream()
|
||||
.map(value -> new ExternalQuarterPrice(Source.GUS, value.year(), value.quarter(),
|
||||
value.market(), city, value.pricePerM2(), now))
|
||||
.toList());
|
||||
});
|
||||
imported += values.size();
|
||||
}
|
||||
}
|
||||
// Publiczne API bez klucza ma limity zapytan - nie spieszymy sie.
|
||||
Thread.sleep(700);
|
||||
} catch (IOException | RuntimeException ex) {
|
||||
log.warn("GUS: nie udalo sie pobrac danych dla {} ({})", city, ex.getMessage());
|
||||
}
|
||||
}
|
||||
log.info("GUS: zaimportowano {} cen kwartalnych", imported);
|
||||
} catch (IOException | RuntimeException ex) {
|
||||
log.warn("GUS: import nieudany ({}) - zostawiam poprzednie dane", ex.getMessage());
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
return imported;
|
||||
}
|
||||
|
||||
private List<String> citiesToTrack() {
|
||||
Map<String, String> cities = new LinkedHashMap<>();
|
||||
repository.findAll().stream()
|
||||
.filter(price -> price.getSource() == Source.NBP)
|
||||
.forEach(price -> cities.putIfAbsent(price.getCityKey(), price.getCity()));
|
||||
listingRepository.findAll().stream()
|
||||
.filter(listing -> listing.getStatus() == ListingStatus.APPROVED)
|
||||
.map(PropertyListing::getCity)
|
||||
.filter(city -> city != null && !city.isBlank())
|
||||
.forEach(city -> cities.putIfAbsent(PriceSnapshotService.key(city), city.trim()));
|
||||
return List.copyOf(cities.values());
|
||||
}
|
||||
|
||||
private InputStream download(String url) throws IOException, InterruptedException {
|
||||
HttpRequest request = HttpRequest.newBuilder(URI.create(url)).timeout(Duration.ofSeconds(60)).GET().build();
|
||||
HttpResponse<byte[]> response = http.send(request, HttpResponse.BodyHandlers.ofByteArray());
|
||||
if (response.statusCode() != 200) {
|
||||
throw new IOException("serwer odpowiedzial " + response.statusCode());
|
||||
}
|
||||
if (response.body().length > MAX_DOWNLOAD_BYTES) {
|
||||
throw new IOException("plik jest za duzy");
|
||||
}
|
||||
return new java.io.ByteArrayInputStream(response.body());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package pl.polskalokalnie.pricehistory;
|
||||
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.EnumType;
|
||||
import jakarta.persistence.Enumerated;
|
||||
import jakarta.persistence.GeneratedValue;
|
||||
import jakarta.persistence.GenerationType;
|
||||
import jakarta.persistence.Id;
|
||||
import jakarta.persistence.Table;
|
||||
import jakarta.persistence.UniqueConstraint;
|
||||
import java.time.Instant;
|
||||
|
||||
/**
|
||||
* Kwartalna srednia cena transakcyjna za m2 mieszkan z zewnetrznego zrodla:
|
||||
* NBP (baza BaRN, 16 miast wojewodzkich i Gdynia) albo GUS (Bank Danych Lokalnych, miasta na prawach powiatu).
|
||||
* Rynek: "Pierwotny", "Wtórny" albo "" (oba razem - tylko GUS).
|
||||
*/
|
||||
@Entity
|
||||
@Table(name = "external_quarter_prices",
|
||||
uniqueConstraints = @UniqueConstraint(columnNames = {"source", "year", "quarter", "market", "cityKey"}))
|
||||
public class ExternalQuarterPrice {
|
||||
|
||||
public enum Source { NBP, GUS }
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@Enumerated(EnumType.STRING)
|
||||
@Column(nullable = false, length = 8)
|
||||
private Source source;
|
||||
|
||||
@Column(nullable = false)
|
||||
private int year;
|
||||
|
||||
@Column(nullable = false)
|
||||
private int quarter;
|
||||
|
||||
@Column(nullable = false, length = 32)
|
||||
private String market;
|
||||
|
||||
@Column(nullable = false, length = 120)
|
||||
private String cityKey;
|
||||
|
||||
@Column(nullable = false, length = 120)
|
||||
private String city;
|
||||
|
||||
@Column(nullable = false)
|
||||
private long pricePerM2;
|
||||
|
||||
@Column(nullable = false)
|
||||
private Instant importedAt;
|
||||
|
||||
protected ExternalQuarterPrice() {
|
||||
}
|
||||
|
||||
public ExternalQuarterPrice(Source source, int year, int quarter, String market, String city, long pricePerM2,
|
||||
Instant importedAt) {
|
||||
this.source = source;
|
||||
this.year = year;
|
||||
this.quarter = quarter;
|
||||
this.market = market;
|
||||
this.city = city;
|
||||
this.cityKey = PriceSnapshotService.key(city);
|
||||
this.pricePerM2 = pricePerM2;
|
||||
this.importedAt = importedAt;
|
||||
}
|
||||
|
||||
public Source getSource() {
|
||||
return source;
|
||||
}
|
||||
|
||||
public int getYear() {
|
||||
return year;
|
||||
}
|
||||
|
||||
public int getQuarter() {
|
||||
return quarter;
|
||||
}
|
||||
|
||||
public String getMarket() {
|
||||
return market;
|
||||
}
|
||||
|
||||
public String getCity() {
|
||||
return city;
|
||||
}
|
||||
|
||||
public String getCityKey() {
|
||||
return cityKey;
|
||||
}
|
||||
|
||||
public long getPricePerM2() {
|
||||
return pricePerM2;
|
||||
}
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package pl.polskalokalnie.pricehistory;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Modifying;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
import pl.polskalokalnie.pricehistory.ExternalQuarterPrice.Source;
|
||||
|
||||
public interface ExternalQuarterPriceRepository extends JpaRepository<ExternalQuarterPrice, Long> {
|
||||
|
||||
List<ExternalQuarterPrice> findByCityKeyAndMarketOrderByYearAscQuarterAsc(String cityKey, String market);
|
||||
|
||||
@Query("select max(p.importedAt) from ExternalQuarterPrice p where p.source = :source")
|
||||
Optional<Instant> findLastImport(@Param("source") Source source);
|
||||
|
||||
@Modifying
|
||||
@Query("delete from ExternalQuarterPrice p where p.source = :source")
|
||||
void deleteBySource(@Param("source") Source source);
|
||||
|
||||
@Modifying
|
||||
@Query("delete from ExternalQuarterPrice p where p.source = :source and p.cityKey = :cityKey")
|
||||
void deleteBySourceAndCityKey(@Param("source") Source source, @Param("cityKey") String cityKey);
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package pl.polskalokalnie.pricehistory;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URLEncoder;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.text.Normalizer;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* Publiczne API GUS - Bank Danych Lokalnych (bdl.stat.gov.pl, bez klucza).
|
||||
* Temat P3794: srednia cena za 1 m2 lokali mieszkalnych sprzedanych w ramach transakcji rynkowych
|
||||
* (dane kwartalne), poziom powiatu - miasta na prawach powiatu maja wlasne dane.
|
||||
*/
|
||||
@Component
|
||||
public class GusBdlClient {
|
||||
|
||||
static final String SUBJECT = "P3794";
|
||||
private static final String API = "https://bdl.stat.gov.pl/api/v1";
|
||||
private static final Pattern QUARTER = Pattern.compile("^(\\d)\\s*kwarta");
|
||||
|
||||
/** Zmienna BDL: kwartal i rynek ("" = oba rynki razem). */
|
||||
public record Variable(long id, int quarter, String market) {
|
||||
}
|
||||
|
||||
public record QuarterValue(int year, int quarter, String market, long pricePerM2) {
|
||||
}
|
||||
|
||||
private final HttpClient http = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(15)).build();
|
||||
private final ObjectMapper mapper = new ObjectMapper();
|
||||
|
||||
/** Id jednostki "Powiat m. <miasto>" / "Powiat m.st. <miasto>" albo null, gdy miasto nie jest na prawach powiatu. */
|
||||
public String cityUnitId(String city) throws IOException, InterruptedException {
|
||||
JsonNode results = get("/units/search?level=5&format=json&page-size=20&name="
|
||||
+ URLEncoder.encode(city, StandardCharsets.UTF_8)).path("results");
|
||||
return matchCityUnit(results, city);
|
||||
}
|
||||
|
||||
public List<Variable> variables() throws IOException, InterruptedException {
|
||||
return parseVariables(get("/variables?subject-id=" + SUBJECT + "&format=json&page-size=100").path("results"));
|
||||
}
|
||||
|
||||
public List<QuarterValue> quarterlyPrices(String unitId, List<Variable> variables, int fromYear, int toYear)
|
||||
throws IOException, InterruptedException {
|
||||
StringBuilder query = new StringBuilder("/data/by-unit/").append(unitId).append("?format=json");
|
||||
variables.forEach(variable -> query.append("&var-id=").append(variable.id()));
|
||||
for (int year = fromYear; year <= toYear; year++) {
|
||||
query.append("&year=").append(year);
|
||||
}
|
||||
return parseUnitData(get(query.toString()), variables);
|
||||
}
|
||||
|
||||
static String matchCityUnit(JsonNode results, String city) {
|
||||
String wanted = normalize(city);
|
||||
for (JsonNode unit : results) {
|
||||
String name = normalize(unit.path("name").asText());
|
||||
// "Powiat m. Kraków", "Powiat m. st. Warszawa" - bez powiatow ziemskich wokol miasta.
|
||||
if (name.equals("powiat m. " + wanted) || name.equals("powiat m. st. " + wanted)
|
||||
|| name.equals("powiat m.st. " + wanted)) {
|
||||
return unit.path("id").asText();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Tylko "ogolem" wg powierzchni lokalu - dla kazdego kwartalu i rynku jedna wartosc.
|
||||
static List<Variable> parseVariables(JsonNode results) {
|
||||
List<Variable> variables = new ArrayList<>();
|
||||
for (JsonNode variable : results) {
|
||||
Matcher quarter = QUARTER.matcher(variable.path("n1").asText());
|
||||
String market = switch (normalize(variable.path("n2").asText())) {
|
||||
case "ogółem" -> "";
|
||||
case "rynek pierwotny" -> "Pierwotny";
|
||||
case "rynek wtórny" -> "Wtórny";
|
||||
default -> null;
|
||||
};
|
||||
if (quarter.find() && market != null && "ogółem".equals(normalize(variable.path("n3").asText()))) {
|
||||
variables.add(new Variable(variable.path("id").asLong(), Integer.parseInt(quarter.group(1)), market));
|
||||
}
|
||||
}
|
||||
return variables;
|
||||
}
|
||||
|
||||
static List<QuarterValue> parseUnitData(JsonNode response, List<Variable> variables) {
|
||||
Map<Long, Variable> byId = new LinkedHashMap<>();
|
||||
variables.forEach(variable -> byId.put(variable.id(), variable));
|
||||
List<QuarterValue> values = new ArrayList<>();
|
||||
for (JsonNode result : response.path("results")) {
|
||||
Variable variable = byId.get(result.path("id").asLong());
|
||||
if (variable == null) {
|
||||
continue;
|
||||
}
|
||||
for (JsonNode value : result.path("values")) {
|
||||
double price = value.path("val").asDouble(0);
|
||||
int year = value.path("year").asInt(0);
|
||||
if (price > 0 && year > 0) {
|
||||
values.add(new QuarterValue(year, variable.quarter(), variable.market(), Math.round(price)));
|
||||
}
|
||||
}
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
private JsonNode get(String path) throws IOException, InterruptedException {
|
||||
HttpRequest request = HttpRequest.newBuilder(URI.create(API + path))
|
||||
.timeout(Duration.ofSeconds(30))
|
||||
.header("Accept", "application/json")
|
||||
.GET()
|
||||
.build();
|
||||
HttpResponse<String> response = http.send(request, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8));
|
||||
if (response.statusCode() != 200) {
|
||||
throw new IOException("GUS BDL odpowiedzial " + response.statusCode() + " dla " + path);
|
||||
}
|
||||
return mapper.readTree(response.body());
|
||||
}
|
||||
|
||||
private static String normalize(String value) {
|
||||
return Normalizer.normalize(value == null ? "" : value.trim(), Normalizer.Form.NFC)
|
||||
.toLowerCase(Locale.ROOT).replaceAll("\\s+", " ");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
package pl.polskalokalnie.pricehistory;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
import javax.xml.XMLConstants;
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.NodeList;
|
||||
|
||||
/**
|
||||
* Odczyt cen transakcyjnych z pliku NBP ceny_mieszkan.xlsx bez zewnetrznych bibliotek (XLSX to ZIP z XML).
|
||||
* Z arkuszy "Rynek pierwotny" i "Rynek wtórny" bierze blok "Ceny transakcyjne": kolumne kwartalu
|
||||
* ("II 2026") i kolumny miast; pomija srednie zbiorcze ("7 miast") oraz indeksy cen.
|
||||
*/
|
||||
public final class NbpXlsxParser {
|
||||
|
||||
public record Row(int year, int quarter, String market, String city, long pricePerM2) {
|
||||
}
|
||||
|
||||
private static final long MAX_ENTRY_BYTES = 20L * 1024 * 1024;
|
||||
private static final Pattern QUARTER = Pattern.compile("^(IV|III|II|I)\\s+(\\d{4})$");
|
||||
private static final Pattern CELL_REF = Pattern.compile("^([A-Z]+)\\d+$");
|
||||
private static final Map<String, String> SHEET_MARKETS = Map.of(
|
||||
"Rynek pierwotny", "Pierwotny",
|
||||
"Rynek wtórny", "Wtórny");
|
||||
|
||||
private NbpXlsxParser() {
|
||||
}
|
||||
|
||||
public static List<Row> parse(InputStream xlsx) throws IOException {
|
||||
Map<String, byte[]> entries = unzip(xlsx);
|
||||
try {
|
||||
List<String> sharedStrings = sharedStrings(entries.get("xl/sharedStrings.xml"));
|
||||
Map<String, String> sheetFiles = sheetFiles(entries);
|
||||
List<Row> rows = new ArrayList<>();
|
||||
for (Map.Entry<String, String> sheet : SHEET_MARKETS.entrySet()) {
|
||||
String file = sheetFiles.get(sheet.getKey());
|
||||
if (file != null && entries.containsKey(file)) {
|
||||
rows.addAll(transactionPrices(read(entries.get(file)), sharedStrings, sheet.getValue()));
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
} catch (IOException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
throw new IOException("Nie udało się odczytać pliku NBP: " + ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static List<Row> transactionPrices(Document sheet, List<String> shared, String market) {
|
||||
List<Map<String, String>> table = new ArrayList<>();
|
||||
NodeList rowNodes = sheet.getElementsByTagName("row");
|
||||
for (int i = 0; i < rowNodes.getLength(); i++) {
|
||||
Map<String, String> cells = new LinkedHashMap<>();
|
||||
NodeList cellNodes = ((Element) rowNodes.item(i)).getElementsByTagName("c");
|
||||
for (int j = 0; j < cellNodes.getLength(); j++) {
|
||||
Element cell = (Element) cellNodes.item(j);
|
||||
Matcher ref = CELL_REF.matcher(cell.getAttribute("r"));
|
||||
String value = cellValue(cell, shared);
|
||||
if (ref.matches() && value != null && !value.isBlank()) {
|
||||
cells.put(ref.group(1), value.trim());
|
||||
}
|
||||
}
|
||||
table.add(cells);
|
||||
}
|
||||
|
||||
String transactionColumn = findColumn(table, "Ceny transakcyjne");
|
||||
Map<String, String> header = table.stream()
|
||||
.filter(row -> row.values().stream().filter("Kwartał"::equals).count() >= 2)
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
if (transactionColumn == null || header == null) {
|
||||
return List.of();
|
||||
}
|
||||
// Kolumna kwartalu bloku transakcyjnego: ostatnia "Kwartał" nie dalej niz naglowek bloku.
|
||||
String quarterColumn = null;
|
||||
for (Map.Entry<String, String> cell : header.entrySet()) {
|
||||
if ("Kwartał".equals(cell.getValue()) && columnIndex(cell.getKey()) <= columnIndex(transactionColumn)) {
|
||||
quarterColumn = cell.getKey();
|
||||
}
|
||||
}
|
||||
if (quarterColumn == null) {
|
||||
return List.of();
|
||||
}
|
||||
Map<String, String> cityColumns = new LinkedHashMap<>();
|
||||
for (int index = columnIndex(quarterColumn) + 1; ; index++) {
|
||||
String name = header.get(columnName(index));
|
||||
if (name == null || name.toLowerCase().contains("miast")) {
|
||||
break;
|
||||
}
|
||||
cityColumns.put(columnName(index), name.replace("*", "").trim());
|
||||
}
|
||||
|
||||
List<Row> result = new ArrayList<>();
|
||||
for (Map<String, String> row : table) {
|
||||
Matcher quarter = QUARTER.matcher(row.getOrDefault(quarterColumn, ""));
|
||||
if (!quarter.matches()) {
|
||||
continue;
|
||||
}
|
||||
int quarterNumber = switch (quarter.group(1)) {
|
||||
case "I" -> 1;
|
||||
case "II" -> 2;
|
||||
case "III" -> 3;
|
||||
default -> 4;
|
||||
};
|
||||
int year = Integer.parseInt(quarter.group(2));
|
||||
cityColumns.forEach((column, city) -> {
|
||||
Double price = number(row.get(column));
|
||||
if (price != null && price > 0) {
|
||||
result.add(new Row(year, quarterNumber, market, city, Math.round(price)));
|
||||
}
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static String findColumn(List<Map<String, String>> table, String text) {
|
||||
for (Map<String, String> row : table) {
|
||||
for (Map.Entry<String, String> cell : row.entrySet()) {
|
||||
if (text.equals(cell.getValue())) {
|
||||
return cell.getKey();
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static String cellValue(Element cell, List<String> shared) {
|
||||
String type = cell.getAttribute("t");
|
||||
if ("inlineStr".equals(type)) {
|
||||
return textOf(cell);
|
||||
}
|
||||
NodeList values = cell.getElementsByTagName("v");
|
||||
if (values.getLength() == 0) {
|
||||
return null;
|
||||
}
|
||||
String raw = values.item(0).getTextContent();
|
||||
if ("s".equals(type)) {
|
||||
int index = Integer.parseInt(raw.trim());
|
||||
return index < shared.size() ? shared.get(index) : null;
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
|
||||
private static List<String> sharedStrings(byte[] xml) throws Exception {
|
||||
List<String> strings = new ArrayList<>();
|
||||
if (xml == null) {
|
||||
return strings;
|
||||
}
|
||||
NodeList items = read(xml).getElementsByTagName("si");
|
||||
for (int i = 0; i < items.getLength(); i++) {
|
||||
strings.add(textOf((Element) items.item(i)));
|
||||
}
|
||||
return strings;
|
||||
}
|
||||
|
||||
private static String textOf(Element element) {
|
||||
StringBuilder text = new StringBuilder();
|
||||
NodeList parts = element.getElementsByTagName("t");
|
||||
for (int i = 0; i < parts.getLength(); i++) {
|
||||
text.append(parts.item(i).getTextContent());
|
||||
}
|
||||
return text.toString();
|
||||
}
|
||||
|
||||
// Nazwa arkusza -> sciezka pliku w ZIP (workbook.xml + relacje).
|
||||
private static Map<String, String> sheetFiles(Map<String, byte[]> entries) throws Exception {
|
||||
Map<String, String> targets = new HashMap<>();
|
||||
NodeList relations = read(entries.get("xl/_rels/workbook.xml.rels")).getElementsByTagName("Relationship");
|
||||
for (int i = 0; i < relations.getLength(); i++) {
|
||||
Element relation = (Element) relations.item(i);
|
||||
String target = relation.getAttribute("Target");
|
||||
targets.put(relation.getAttribute("Id"), target.startsWith("/") ? target.substring(1) : "xl/" + target);
|
||||
}
|
||||
Map<String, String> files = new HashMap<>();
|
||||
NodeList sheets = read(entries.get("xl/workbook.xml")).getElementsByTagName("sheet");
|
||||
for (int i = 0; i < sheets.getLength(); i++) {
|
||||
Element sheet = (Element) sheets.item(i);
|
||||
files.put(sheet.getAttribute("name").trim(), targets.get(sheet.getAttribute("r:id")));
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
private static Map<String, byte[]> unzip(InputStream input) throws IOException {
|
||||
Map<String, byte[]> entries = new HashMap<>();
|
||||
try (ZipInputStream zip = new ZipInputStream(input)) {
|
||||
ZipEntry entry;
|
||||
while ((entry = zip.getNextEntry()) != null) {
|
||||
if (entry.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
byte[] data = zip.readNBytes((int) MAX_ENTRY_BYTES + 1);
|
||||
if (data.length > MAX_ENTRY_BYTES) {
|
||||
throw new IOException("Plik NBP ma nieoczekiwanie duzy element: " + entry.getName());
|
||||
}
|
||||
entries.put(entry.getName(), data);
|
||||
}
|
||||
}
|
||||
if (!entries.containsKey("xl/workbook.xml")) {
|
||||
throw new IOException("To nie jest plik XLSX");
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
private static Document read(byte[] xml) throws Exception {
|
||||
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
|
||||
// Plik z zewnatrz - bez encji zewnetrznych i DTD (ochrona przed XXE).
|
||||
factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
|
||||
factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
|
||||
factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
|
||||
factory.setExpandEntityReferences(false);
|
||||
DocumentBuilder builder = factory.newDocumentBuilder();
|
||||
return builder.parse(new ByteArrayInputStream(xml));
|
||||
}
|
||||
|
||||
private static Double number(String value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return Double.parseDouble(value.replace(',', '.'));
|
||||
} catch (NumberFormatException ex) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static int columnIndex(String column) {
|
||||
int index = 0;
|
||||
for (char letter : column.toCharArray()) {
|
||||
index = index * 26 + (letter - 'A' + 1);
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
private static String columnName(int index) {
|
||||
StringBuilder name = new StringBuilder();
|
||||
while (index > 0) {
|
||||
int remainder = (index - 1) % 26;
|
||||
name.insert(0, (char) ('A' + remainder));
|
||||
index = (index - 1) / 26;
|
||||
}
|
||||
return name.toString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package pl.polskalokalnie.pricehistory;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.util.List;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import pl.polskalokalnie.listing.OfferType;
|
||||
import pl.polskalokalnie.listing.PropertyType;
|
||||
|
||||
/** Publiczna historia srednich cen ofertowych za m2 dla miasta lub dzielnicy. */
|
||||
@RestController
|
||||
@RequestMapping("/api/price-history")
|
||||
public class PriceHistoryController {
|
||||
|
||||
private final PriceSnapshotRepository snapshotRepository;
|
||||
private final PriceSnapshotService snapshotService;
|
||||
private final ExternalQuarterPriceRepository externalRepository;
|
||||
|
||||
public PriceHistoryController(PriceSnapshotRepository snapshotRepository, PriceSnapshotService snapshotService,
|
||||
ExternalQuarterPriceRepository externalRepository) {
|
||||
this.snapshotRepository = snapshotRepository;
|
||||
this.snapshotService = snapshotService;
|
||||
this.externalRepository = externalRepository;
|
||||
}
|
||||
|
||||
public record PricePoint(LocalDate date, long avgPricePerM2, int listingsCount) {
|
||||
}
|
||||
|
||||
public record QuarterPoint(int year, int quarter, long pricePerM2) {
|
||||
}
|
||||
|
||||
/**
|
||||
* points - dzienne zapisy od "from"; current - srednia z ogloszen w tej chwili (null, gdy brak);
|
||||
* trackedSince - od kiedy serwis w ogole zbiera historie (null, gdy jeszcze nic nie zapisano);
|
||||
* nbp / gus - kwartalne ceny transakcyjne mieszkan dla calego miasta (puste, gdy zrodlo go nie obejmuje);
|
||||
* nbpMarket - rynek, ktorego dotyczy nbp (NBP nie podaje obu rynkow razem - wtedy wtorny).
|
||||
*/
|
||||
public record PriceHistoryResponse(List<PricePoint> points, PricePoint current, LocalDate trackedSince,
|
||||
List<QuarterPoint> nbp, List<QuarterPoint> gus, String nbpMarket) {
|
||||
}
|
||||
|
||||
@GetMapping
|
||||
public PriceHistoryResponse history(
|
||||
@RequestParam String city,
|
||||
@RequestParam(required = false, defaultValue = "") String district,
|
||||
@RequestParam(required = false, defaultValue = "SALE") OfferType offerType,
|
||||
@RequestParam(required = false, defaultValue = "APARTMENT") PropertyType propertyType,
|
||||
@RequestParam(required = false, defaultValue = "") String market,
|
||||
@RequestParam(required = false) LocalDate from
|
||||
) {
|
||||
LocalDate since = from != null ? from : PriceSnapshotService.today().minusYears(1);
|
||||
String wantedMarket = market.trim();
|
||||
List<PricePoint> points = snapshotRepository
|
||||
.findByCityKeyAndDistrictKeyAndOfferTypeAndPropertyTypeAndMarketAndSnapshotDateGreaterThanEqualOrderBySnapshotDateAsc(
|
||||
PriceSnapshotService.key(city), PriceSnapshotService.key(district), offerType, propertyType,
|
||||
wantedMarket, since)
|
||||
.stream()
|
||||
.map(snapshot -> new PricePoint(snapshot.getSnapshotDate(), snapshot.getAvgPricePerM2(), snapshot.getListingsCount()))
|
||||
.toList();
|
||||
PriceSnapshotService.Average now = snapshotService.currentAverage(offerType, propertyType, wantedMarket, city, district);
|
||||
PricePoint current = now == null ? null
|
||||
: new PricePoint(PriceSnapshotService.today(), now.avgPricePerM2(), now.listingsCount());
|
||||
// NBP i GUS podaja ceny transakcyjne mieszkan - dla domow i najmu nie ma porownania.
|
||||
boolean comparable = offerType == OfferType.SALE && propertyType == PropertyType.APARTMENT;
|
||||
String nbpMarket = wantedMarket.isEmpty() ? "Wtórny" : wantedMarket;
|
||||
String cityKey = PriceSnapshotService.key(city);
|
||||
List<QuarterPoint> nbp = comparable ? quarters(cityKey, nbpMarket, ExternalQuarterPrice.Source.NBP) : List.of();
|
||||
List<QuarterPoint> gus = comparable ? quarters(cityKey, wantedMarket, ExternalQuarterPrice.Source.GUS) : List.of();
|
||||
return new PriceHistoryResponse(points, current, snapshotRepository.findFirstSnapshotDate().orElse(null),
|
||||
nbp, gus, nbpMarket);
|
||||
}
|
||||
|
||||
private List<QuarterPoint> quarters(String cityKey, String market, ExternalQuarterPrice.Source source) {
|
||||
return externalRepository.findByCityKeyAndMarketOrderByYearAscQuarterAsc(cityKey, market).stream()
|
||||
.filter(price -> price.getSource() == source)
|
||||
.map(price -> new QuarterPoint(price.getYear(), price.getQuarter(), price.getPricePerM2()))
|
||||
.toList();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package pl.polskalokalnie.pricehistory;
|
||||
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.EnumType;
|
||||
import jakarta.persistence.Enumerated;
|
||||
import jakarta.persistence.GeneratedValue;
|
||||
import jakarta.persistence.GenerationType;
|
||||
import jakarta.persistence.Id;
|
||||
import jakarta.persistence.Index;
|
||||
import jakarta.persistence.Table;
|
||||
import jakarta.persistence.UniqueConstraint;
|
||||
import java.time.LocalDate;
|
||||
import pl.polskalokalnie.listing.OfferType;
|
||||
import pl.polskalokalnie.listing.PropertyType;
|
||||
|
||||
/**
|
||||
* Dzienny zapis sredniej ceny ofertowej za m2 z opublikowanych ogloszen - zrodlo historii cen.
|
||||
* Jeden wiersz na (dzien, typ oferty, typ nieruchomosci, rynek, miasto, dzielnica).
|
||||
* Pusta dzielnica = cale miasto, pusty rynek = wtorny i pierwotny razem.
|
||||
* Klucze (cityKey, districtKey) sa znormalizowane, zeby "Kraków" i " kraków" trafialy w to samo.
|
||||
*/
|
||||
@Entity
|
||||
@Table(name = "price_snapshots",
|
||||
uniqueConstraints = @UniqueConstraint(columnNames = {
|
||||
"snapshotDate", "offerType", "propertyType", "market", "cityKey", "districtKey"}),
|
||||
indexes = @Index(columnList = "cityKey, districtKey, offerType, propertyType, market, snapshotDate"))
|
||||
public class PriceSnapshot {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@Column(nullable = false)
|
||||
private LocalDate snapshotDate;
|
||||
|
||||
@Enumerated(EnumType.STRING)
|
||||
@Column(nullable = false, length = 16)
|
||||
private OfferType offerType;
|
||||
|
||||
@Enumerated(EnumType.STRING)
|
||||
@Column(nullable = false, length = 16)
|
||||
private PropertyType propertyType;
|
||||
|
||||
@Column(nullable = false, length = 32)
|
||||
private String market;
|
||||
|
||||
@Column(nullable = false, length = 120)
|
||||
private String cityKey;
|
||||
|
||||
@Column(nullable = false, length = 120)
|
||||
private String districtKey;
|
||||
|
||||
@Column(nullable = false, length = 120)
|
||||
private String city;
|
||||
|
||||
@Column(nullable = false, length = 120)
|
||||
private String district;
|
||||
|
||||
@Column(nullable = false)
|
||||
private int listingsCount;
|
||||
|
||||
@Column(nullable = false)
|
||||
private long avgPricePerM2;
|
||||
|
||||
protected PriceSnapshot() {
|
||||
}
|
||||
|
||||
public PriceSnapshot(LocalDate snapshotDate, OfferType offerType, PropertyType propertyType, String market,
|
||||
String city, String district, int listingsCount, long avgPricePerM2) {
|
||||
this.snapshotDate = snapshotDate;
|
||||
this.offerType = offerType;
|
||||
this.propertyType = propertyType;
|
||||
this.market = market;
|
||||
this.city = city;
|
||||
this.district = district;
|
||||
this.cityKey = PriceSnapshotService.key(city);
|
||||
this.districtKey = PriceSnapshotService.key(district);
|
||||
this.listingsCount = listingsCount;
|
||||
this.avgPricePerM2 = avgPricePerM2;
|
||||
}
|
||||
|
||||
public LocalDate getSnapshotDate() {
|
||||
return snapshotDate;
|
||||
}
|
||||
|
||||
public int getListingsCount() {
|
||||
return listingsCount;
|
||||
}
|
||||
|
||||
public long getAvgPricePerM2() {
|
||||
return avgPricePerM2;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package pl.polskalokalnie.pricehistory;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Modifying;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
import pl.polskalokalnie.listing.OfferType;
|
||||
import pl.polskalokalnie.listing.PropertyType;
|
||||
|
||||
public interface PriceSnapshotRepository extends JpaRepository<PriceSnapshot, Long> {
|
||||
|
||||
boolean existsBySnapshotDate(LocalDate snapshotDate);
|
||||
|
||||
@Modifying
|
||||
@Query("delete from PriceSnapshot s where s.snapshotDate = :date")
|
||||
void deleteBySnapshotDate(@Param("date") LocalDate date);
|
||||
|
||||
List<PriceSnapshot> findByCityKeyAndDistrictKeyAndOfferTypeAndPropertyTypeAndMarketAndSnapshotDateGreaterThanEqualOrderBySnapshotDateAsc(
|
||||
String cityKey, String districtKey, OfferType offerType, PropertyType propertyType, String market, LocalDate from);
|
||||
|
||||
@Query("select min(s.snapshotDate) from PriceSnapshot s")
|
||||
Optional<LocalDate> findFirstSnapshotDate();
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package pl.polskalokalnie.pricehistory;
|
||||
|
||||
import java.text.Normalizer;
|
||||
import java.time.LocalDate;
|
||||
import java.time.ZoneId;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import pl.polskalokalnie.listing.ListingRepository;
|
||||
import pl.polskalokalnie.listing.ListingStatus;
|
||||
import pl.polskalokalnie.listing.OfferType;
|
||||
import pl.polskalokalnie.listing.PropertyListing;
|
||||
import pl.polskalokalnie.listing.PropertyType;
|
||||
|
||||
/**
|
||||
* Historia cen: codzienny zapis srednich cen ofertowych za m2 z opublikowanych ogloszen.
|
||||
* Danych sprzed pierwszego zapisu nie odtwarzamy - historia rosnie z kazdym dniem.
|
||||
*/
|
||||
@Service
|
||||
public class PriceSnapshotService {
|
||||
|
||||
public static final ZoneId ZONE = ZoneId.of("Europe/Warsaw");
|
||||
/** Wartosc rynku oznaczajaca "wtorny i pierwotny razem". */
|
||||
public static final String ALL_MARKETS = "";
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(PriceSnapshotService.class);
|
||||
|
||||
private final ListingRepository listingRepository;
|
||||
private final PriceSnapshotRepository snapshotRepository;
|
||||
|
||||
public PriceSnapshotService(ListingRepository listingRepository, PriceSnapshotRepository snapshotRepository) {
|
||||
this.listingRepository = listingRepository;
|
||||
this.snapshotRepository = snapshotRepository;
|
||||
}
|
||||
|
||||
/** Srednia na dany moment - ta sama grupa, co w zapisie dziennym. */
|
||||
public record Average(long avgPricePerM2, int listingsCount) {
|
||||
}
|
||||
|
||||
private record GroupKey(OfferType offerType, PropertyType propertyType, String market, String city, String district) {
|
||||
}
|
||||
|
||||
// Co noc zapisujemy stan z konca dnia; przy starcie uzupelniamy dzisiejszy zapis, jesli go brakuje.
|
||||
// Transakcja na tych metodach, bo wywolanie recordToday() z wnetrza klasy omija proxy Springa.
|
||||
@Scheduled(cron = "0 30 2 * * *", zone = "Europe/Warsaw")
|
||||
@Transactional
|
||||
public void recordDaily() {
|
||||
recordToday();
|
||||
}
|
||||
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
@Transactional
|
||||
public void recordOnStartupIfMissing() {
|
||||
if (!snapshotRepository.existsBySnapshotDate(today())) {
|
||||
recordToday();
|
||||
}
|
||||
}
|
||||
|
||||
/** Zapis dzisiejszych srednich; ponowne wywolanie w tym samym dniu nadpisuje dzisiejsze wiersze. */
|
||||
@Transactional
|
||||
public int recordToday() {
|
||||
LocalDate date = today();
|
||||
snapshotRepository.deleteBySnapshotDate(date);
|
||||
Map<GroupKey, List<Double>> groups = group(pricedListings());
|
||||
List<PriceSnapshot> rows = new ArrayList<>();
|
||||
groups.forEach((key, prices) -> rows.add(new PriceSnapshot(date, key.offerType(), key.propertyType(), key.market(),
|
||||
key.city(), key.district(), prices.size(), Math.round(average(prices)))));
|
||||
snapshotRepository.saveAll(rows);
|
||||
log.info("Historia cen: zapisano {} srednich za {}", rows.size(), date);
|
||||
return rows.size();
|
||||
}
|
||||
|
||||
/** Aktualna srednia liczona wprost z ogloszen - "dzisiaj" na wykresie jest zawsze swieze. */
|
||||
public Average currentAverage(OfferType offerType, PropertyType propertyType, String market, String city, String district) {
|
||||
String wantedCity = key(city);
|
||||
String wantedDistrict = key(district);
|
||||
List<Double> prices = pricedListings().stream()
|
||||
.filter(listing -> listing.getOfferType() == offerType && listing.getPropertyType() == propertyType)
|
||||
.filter(listing -> key(listing.getCity()).equals(wantedCity))
|
||||
.filter(listing -> wantedDistrict.isEmpty() || key(listing.getDistrict()).equals(wantedDistrict))
|
||||
.filter(listing -> market.isEmpty() || market.equalsIgnoreCase(trim(listing.getMarket())))
|
||||
.map(PriceSnapshotService::pricePerM2)
|
||||
.toList();
|
||||
return prices.isEmpty() ? null : new Average(Math.round(average(prices)), prices.size());
|
||||
}
|
||||
|
||||
private List<PropertyListing> pricedListings() {
|
||||
return listingRepository.findAll().stream()
|
||||
.filter(listing -> listing.getStatus() == ListingStatus.APPROVED)
|
||||
.filter(listing -> listing.getOfferType() != null && listing.getPropertyType() != null)
|
||||
.filter(listing -> listing.getPrice() != null && listing.getPrice().doubleValue() > 0)
|
||||
.filter(listing -> listing.getArea() != null && listing.getArea() > 0)
|
||||
.filter(listing -> !trim(listing.getCity()).isEmpty())
|
||||
.toList();
|
||||
}
|
||||
|
||||
// Kazde ogloszenie liczy sie do miasta i (jesli ma) do dzielnicy - dla wszystkich rynkow i dla swojego rynku.
|
||||
private static Map<GroupKey, List<Double>> group(List<PropertyListing> listings) {
|
||||
Map<GroupKey, List<Double>> groups = new LinkedHashMap<>();
|
||||
for (PropertyListing listing : listings) {
|
||||
double price = pricePerM2(listing);
|
||||
String city = trim(listing.getCity());
|
||||
String district = trim(listing.getDistrict());
|
||||
String market = trim(listing.getMarket());
|
||||
List<String> markets = market.isEmpty() ? List.of(ALL_MARKETS) : List.of(ALL_MARKETS, market);
|
||||
List<String> districts = district.isEmpty() ? List.of("") : List.of("", district);
|
||||
for (String groupMarket : markets) {
|
||||
for (String groupDistrict : districts) {
|
||||
GroupKey key = new GroupKey(listing.getOfferType(), listing.getPropertyType(), groupMarket,
|
||||
city, groupDistrict);
|
||||
groups.computeIfAbsent(canonical(groups, key), ignored -> new ArrayList<>()).add(price);
|
||||
}
|
||||
}
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
|
||||
// Ta sama grupa niezaleznie od wielkosci liter i spacji - pierwsza napotkana pisownia zostaje nazwa.
|
||||
private static GroupKey canonical(Map<GroupKey, List<Double>> groups, GroupKey candidate) {
|
||||
for (GroupKey existing : groups.keySet()) {
|
||||
if (existing.offerType() == candidate.offerType() && existing.propertyType() == candidate.propertyType()
|
||||
&& existing.market().equalsIgnoreCase(candidate.market())
|
||||
&& key(existing.city()).equals(key(candidate.city()))
|
||||
&& key(existing.district()).equals(key(candidate.district()))) {
|
||||
return existing;
|
||||
}
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
public static LocalDate today() {
|
||||
return LocalDate.now(ZONE);
|
||||
}
|
||||
|
||||
/** Klucz porownania nazw: male litery, bez nadmiarowych spacji (polskie znaki zostaja). */
|
||||
public static String key(String value) {
|
||||
return Normalizer.normalize(trim(value), Normalizer.Form.NFC).toLowerCase(Locale.ROOT).replaceAll("\\s+", " ");
|
||||
}
|
||||
|
||||
private static String trim(String value) {
|
||||
return value == null ? "" : value.trim();
|
||||
}
|
||||
|
||||
private static double pricePerM2(PropertyListing listing) {
|
||||
return listing.getPrice().doubleValue() / listing.getArea();
|
||||
}
|
||||
|
||||
private static double average(List<Double> values) {
|
||||
return values.stream().mapToDouble(Double::doubleValue).average().orElse(0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package pl.polskalokalnie.pricehistory;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import java.util.List;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
// Probki w formacie odpowiedzi bdl.stat.gov.pl/api/v1 (sprawdzonym 2026-09-22).
|
||||
class GusBdlClientTest {
|
||||
|
||||
private final ObjectMapper mapper = new ObjectMapper();
|
||||
|
||||
@Test
|
||||
void matchesOnlyCityWithPowiatRights() throws Exception {
|
||||
var units = mapper.readTree("""
|
||||
[{"id":"071412865000","name":"Powiat m. st. Warszawa"},
|
||||
{"id":"011212106000","name":"Powiat krakowski"},
|
||||
{"id":"011212161000","name":"Powiat m. Kraków"}]""");
|
||||
|
||||
assertThat(GusBdlClient.matchCityUnit(units, "Warszawa")).isEqualTo("071412865000");
|
||||
assertThat(GusBdlClient.matchCityUnit(units, "kraków")).isEqualTo("011212161000");
|
||||
assertThat(GusBdlClient.matchCityUnit(units, "Zabierzów")).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void readsQuarterlyTotalsPerMarket() throws Exception {
|
||||
var variables = GusBdlClient.parseVariables(mapper.readTree("""
|
||||
[{"id":634007,"n1":"1 kwartał","n2":"ogółem","n3":"ogółem"},
|
||||
{"id":634008,"n1":"1 kwartał","n2":"ogółem","n3":"do 40 m2"},
|
||||
{"id":634012,"n1":"1 kwartał","n2":"rynek pierwotny","n3":"ogółem"},
|
||||
{"id":634032,"n1":"2 kwartał","n2":"rynek wtórny","n3":"ogółem"}]"""));
|
||||
|
||||
assertThat(variables).containsExactly(
|
||||
new GusBdlClient.Variable(634007, 1, ""),
|
||||
new GusBdlClient.Variable(634012, 1, "Pierwotny"),
|
||||
new GusBdlClient.Variable(634032, 2, "Wtórny"));
|
||||
|
||||
List<GusBdlClient.QuarterValue> values = GusBdlClient.parseUnitData(mapper.readTree("""
|
||||
{"results":[
|
||||
{"id":634007,"values":[{"year":"2024","val":13210.4},{"year":"2025","val":0}]},
|
||||
{"id":634032,"values":[{"year":"2024","val":14135}]},
|
||||
{"id":999999,"values":[{"year":"2024","val":1}]}]}"""), variables);
|
||||
|
||||
assertThat(values).containsExactly(
|
||||
new GusBdlClient.QuarterValue(2024, 1, "", 13210),
|
||||
new GusBdlClient.QuarterValue(2024, 2, "Wtórny", 14135));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package pl.polskalokalnie.pricehistory;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.util.List;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class NbpXlsxParserTest {
|
||||
|
||||
// Plik NBP (ceny_mieszkan.xlsx) pobrany 2026-09-22 - prawdziwy uklad arkuszy.
|
||||
private List<NbpXlsxParser.Row> parseFixture() throws Exception {
|
||||
try (InputStream xlsx = getClass().getResourceAsStream("/nbp/ceny_mieszkan.xlsx")) {
|
||||
return NbpXlsxParser.parse(xlsx);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void readsTransactionPricesPerCityAndMarket() throws Exception {
|
||||
List<NbpXlsxParser.Row> rows = parseFixture();
|
||||
|
||||
// Wartosci z arkusza "Rynek pierwotny", blok "Ceny transakcyjne", III kw. 2006.
|
||||
assertThat(rows).contains(
|
||||
new NbpXlsxParser.Row(2006, 3, "Pierwotny", "Kraków", 6816),
|
||||
new NbpXlsxParser.Row(2006, 3, "Pierwotny", "Białystok", 3008));
|
||||
// Oba rynki, "Gdynia**" bez gwiazdek, najnowszy kwartal w pliku.
|
||||
assertThat(rows).anyMatch(row -> row.market().equals("Wtórny") && row.city().equals("Warszawa")
|
||||
&& row.year() == 2026 && row.quarter() == 2);
|
||||
assertThat(rows).anyMatch(row -> row.city().equals("Gdynia"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void skipsAggregatesAndOfferPrices() throws Exception {
|
||||
List<NbpXlsxParser.Row> rows = parseFixture();
|
||||
|
||||
assertThat(rows).noneMatch(row -> row.city().contains("miast"));
|
||||
// Cena ofertowa Krakowa na rynku pierwotnym w III kw. 2006 to 6021 - nie moze trafic do transakcyjnych.
|
||||
assertThat(rows).doesNotContain(new NbpXlsxParser.Row(2006, 3, "Pierwotny", "Kraków", 6021));
|
||||
assertThat(rows.stream().map(NbpXlsxParser.Row::city).distinct()).hasSize(17);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package pl.polskalokalnie.pricehistory;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.List;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import pl.polskalokalnie.listing.ListingRepository;
|
||||
import pl.polskalokalnie.listing.ListingStatus;
|
||||
import pl.polskalokalnie.listing.OfferType;
|
||||
import pl.polskalokalnie.listing.PropertyListing;
|
||||
import pl.polskalokalnie.listing.PropertyType;
|
||||
|
||||
class PriceSnapshotServiceTest {
|
||||
|
||||
private final ListingRepository listings = mock(ListingRepository.class);
|
||||
private final PriceSnapshotRepository snapshots = mock(PriceSnapshotRepository.class);
|
||||
private final PriceSnapshotService service = new PriceSnapshotService(listings, snapshots);
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
void recordsCityDistrictAndMarketAverages() {
|
||||
when(listings.findAll()).thenReturn(List.of(
|
||||
listing("Kraków", "Podgórze", "Wtórny", 1_000_000, 100, ListingStatus.APPROVED), // 10 000 zl/m2
|
||||
listing(" kraków ", "podgórze", "Pierwotny", 600_000, 50, ListingStatus.APPROVED), // 12 000 zl/m2
|
||||
listing("Kraków", null, null, 800_000, 100, ListingStatus.APPROVED), // 8 000 zl/m2
|
||||
listing("Kraków", "Podgórze", "Wtórny", 9_000_000, 10, ListingStatus.PENDING))); // nieopublikowane
|
||||
|
||||
service.recordToday();
|
||||
|
||||
ArgumentCaptor<List<PriceSnapshot>> saved = ArgumentCaptor.forClass(List.class);
|
||||
verify(snapshots).saveAll(saved.capture());
|
||||
List<PriceSnapshot> rows = saved.getValue();
|
||||
|
||||
// Cale miasto, wszystkie rynki: (10 000 + 12 000 + 8 000) / 3.
|
||||
assertThat(find(rows, "", "")).extracting(PriceSnapshot::getAvgPricePerM2, PriceSnapshot::getListingsCount)
|
||||
.containsExactly(10_000L, 3);
|
||||
// Dzielnica bez wzgledu na wielkosc liter: (10 000 + 12 000) / 2.
|
||||
assertThat(find(rows, "podgórze", "")).extracting(PriceSnapshot::getAvgPricePerM2, PriceSnapshot::getListingsCount)
|
||||
.containsExactly(11_000L, 2);
|
||||
// Tylko rynek wtorny w miescie.
|
||||
assertThat(find(rows, "", "Wtórny")).extracting(PriceSnapshot::getAvgPricePerM2, PriceSnapshot::getListingsCount)
|
||||
.containsExactly(10_000L, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void currentAverageUsesOnlyMatchingPublishedListings() {
|
||||
when(listings.findAll()).thenReturn(List.of(
|
||||
listing("Kraków", "Podgórze", "Wtórny", 1_000_000, 100, ListingStatus.APPROVED),
|
||||
listing("Kraków", "Podgórze", "Wtórny", 9_000_000, 10, ListingStatus.PAUSED),
|
||||
listing("Warszawa", "Mokotów", "Wtórny", 2_000_000, 100, ListingStatus.APPROVED)));
|
||||
|
||||
PriceSnapshotService.Average average = service.currentAverage(
|
||||
OfferType.SALE, PropertyType.APARTMENT, "", "KRAKÓW", "");
|
||||
|
||||
assertThat(average).isEqualTo(new PriceSnapshotService.Average(10_000L, 1));
|
||||
assertThat(service.currentAverage(OfferType.SALE, PropertyType.APARTMENT, "", "Gdańsk", "")).isNull();
|
||||
}
|
||||
|
||||
private static PriceSnapshot find(List<PriceSnapshot> rows, String districtKey, String market) {
|
||||
return rows.stream()
|
||||
.filter(row -> readField(row, "districtKey").equals(districtKey) && readField(row, "market").equals(market))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new AssertionError("brak wiersza " + districtKey + "/" + market));
|
||||
}
|
||||
|
||||
private static String readField(PriceSnapshot row, String name) {
|
||||
try {
|
||||
var field = PriceSnapshot.class.getDeclaredField(name);
|
||||
field.setAccessible(true);
|
||||
return (String) field.get(row);
|
||||
} catch (ReflectiveOperationException ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static PropertyListing listing(String city, String district, String market, int price, double area,
|
||||
ListingStatus status) {
|
||||
PropertyListing listing = new PropertyListing();
|
||||
listing.setOfferType(OfferType.SALE);
|
||||
listing.setPropertyType(PropertyType.APARTMENT);
|
||||
listing.setCity(city);
|
||||
listing.setDistrict(district);
|
||||
listing.setMarket(market);
|
||||
listing.setPrice(BigDecimal.valueOf(price));
|
||||
listing.setArea(area);
|
||||
listing.setStatus(status);
|
||||
return listing;
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Reference in New Issue
Block a user