david-spacil/aiscr-qgis-amcr-viewer
1
0
Fork 0
mirror of https://github.com/ARUP-CAS/aiscr-qgis-amcr-viewer.git synced 2026-08-14 19:09:00 +02:00
Code Issues Packages Projects Releases Wiki Activity

feature: update funkcí pro stahování dat

Browse Source 
funkce fetch_set a download_heslare byly upraveny pro stahování dat nejen z OAI-PMH API, ale nově i z API digiarchivu (= osoby se nově nestahují z hesláře osob, ale z facetek digiarchivu, kde mají osoby přidělené "role" nálezce/vedoucí)
This commit is contained in:
David Spáčil
2026-06-29 13:48:23 +02:00
parent 66532c108b
commit e84ee8e543
1 changed files with 100 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
amcr_viewer/amcr_codelists.py
+98 -67
View File
@@ -103,13 +103,19 @@ def load_all_data():
return categorized_data return categorized_data
def fetch_set(internal_name, api_set, task=None): def fetch_set(base_url, internal_name, api_set, task=None):
dataset = [] dataset = []
params = { params_amcr = {
"verb": "ListRecords", "verb": "ListRecords",
"metadataPrefix": "oai_dc", "metadataPrefix": "oai_dc",
"set": api_set "set": api_set
} }
params_da = {
"entity": "samostatny_nalez" if internal_name == "nalezce" else "akce",
"rows": 0,
"noFacets": "false",
"onlyFacets": "true"
}
while True: while True:
# Check for cancellation at each iteration # Check for cancellation at each iteration
@@ -117,74 +123,94 @@ def fetch_set(internal_name, api_set, task=None):
return None return None
try: try:
response = requests.get(BASE_URL, params=params, timeout=30) if "digiarchiv" not in base_url:
response.raise_for_status() response = requests.get(base_url, params=params_amcr, timeout=30)
root = ET.fromstring(response.content) # nosec response.raise_for_status()
root = ET.fromstring(response.content) # nosec
records = root.findall('.//oai:record', NS) records = root.findall('.//oai:record', NS)
for rec in records: for rec in records:
metadata = rec.find('.//oai_dc:dc', NS) metadata = rec.find('.//oai_dc:dc', NS)
if metadata is not None: if metadata is not None:
# Code (identifier) # Code (identifier)
identifier_el = metadata.find('dc:identifier', NS) identifier_el = metadata.find('dc:identifier', NS)
kod = ( kod = (
identifier_el.text identifier_el.text
if identifier_el is not None if identifier_el is not None
else "" else ""
)
# Title – filter out system labels "AMČR - ..."
titles = metadata.findall('dc:title', NS)
nazev = ""
for t in titles:
if (
t.text
and not t.text.startswith("AMČR -")
and not t.text.startswith(" AMČR -")
):
nazev = t.text
break
# If no title passed the filter, fall back
# to the first available one
if not nazev and titles:
nazev = titles[0].text
specialni_pripady = ['okres', 'katastr']
if internal_name in specialni_pripady:
kod = nazev
if internal_name == 'pristupnost':
kod = next(
(
t.text for t in titles
if t.text
and len(t.text) == 1
and t.text.isalpha()
),
None
) )
# Skip records without a valid one-letter code –
# a None code would end up in the CSV and later # Title – filter out system labels "AMČR - ..."
# in the API filter as the string "None" titles = metadata.findall('dc:title', NS)
if not kod: nazev = ""
continue for t in titles:
if (
t.text
and not t.text.startswith("AMČR -")
and not t.text.startswith(" AMČR -")
):
nazev = t.text
break
# If no title passed the filter, fall back
# to the first available one
if not nazev and titles:
nazev = titles[0].text
specialni_pripady = ['okres', 'katastr']
if internal_name in specialni_pripady:
kod = nazev
if internal_name == 'pristupnost':
kod = next(
(
t.text for t in titles
if t.text
and len(t.text) == 1
and t.text.isalpha()
),
None
)
# Skip records without a valid one-letter code –
# a None code would end up in the CSV and later
# in the API filter as the string "None"
if not kod:
continue
dataset.append({
'Název': nazev,
'Kód': kod,
'Kategorie': internal_name
})
# Pagination
token = root.find('.//oai:resumptionToken', NS)
if token is not None and token.text:
params_amcr = {
"verb": "ListRecords",
"resumptionToken": token.text
}
time.sleep(0.5)
else:
break
else:
response = requests.get(base_url, params=params_da, timeout=30)
response.raise_for_status()
data_json = response.json()
records = data_json['facet_counts']['facet_fields'][api_set]
for r in records:
nazev = r["name"]
dataset.append({ dataset.append({
'Název': nazev, 'Název': nazev,
'Kód': kod, 'Kód': nazev,
'Kategorie': internal_name 'Kategorie': internal_name
}) })
# Pagination
token = root.find('.//oai:resumptionToken', NS)
if token is not None and token.text:
params = {
"verb": "ListRecords",
"resumptionToken": token.text
}
time.sleep(0.5)
else:
break break
except Exception as e: except Exception as e:
@@ -201,8 +227,13 @@ def download_heslare(task=None):
ensure_codelists_dir() ensure_codelists_dir()
all_data = [] all_data = []
total_sets = len(slovnicek) total_sets = len(slovnicek)
# index, (interni, api_nazev)
for index, (key, value) in enumerate(slovnicek.items()):
base_url = value[0]
interni = key
api_nazev = value[1]
for index, (interni, api_nazev) in enumerate(slovnicek.items()):
# Check if the user cancelled the task via the QGIS taskbar # Check if the user cancelled the task via the QGIS taskbar
if task and task.isCanceled(): if task and task.isCanceled():
return False return False
@@ -212,7 +243,7 @@ def download_heslare(task=None):
"AMČR", Qgis.Info) "AMČR", Qgis.Info)
# Pass the task correctly to the updated fetch function # Pass the task correctly to the updated fetch function
data = fetch_set(interni, api_nazev, task=task) data = fetch_set(base_url, interni, api_nazev, task=task)
if data is None: if data is None:
return False # Cancelled mid-download return False # Cancelled mid-download