504 lines
No EOL
23 KiB
Python
504 lines
No EOL
23 KiB
Python
# @fileoverview Automatisierter Web-Scraper für SAP Digital Manufacturing Release-Daten und API-Matrizen.
|
|
# Nutzt Selenium WebDriver zur dynamischen Inhaltsextraktion und Pandas zur Datentransformation.
|
|
# @author MBT
|
|
# @version 1.1
|
|
|
|
import time
|
|
import os
|
|
import re
|
|
import json
|
|
import pandas as pd
|
|
from dotenv import load_dotenv
|
|
from selenium import webdriver
|
|
from selenium.webdriver.edge.service import Service
|
|
from selenium.webdriver.edge.options import Options
|
|
from selenium.webdriver.common.by import By
|
|
from selenium.webdriver.support.ui import WebDriverWait
|
|
from selenium.webdriver.support import expected_conditions as EC
|
|
|
|
|
|
load_dotenv()
|
|
|
|
def clean_endpoint_path(path:str):
|
|
"""
|
|
Bereinigt einen extrahierten API-Endpoint-Pfad von anhängenden Satzzeichen und Platzhaltern.
|
|
Schneidet alles ab dem Zeichen '<' ab und entfernt Suffixe wie '.', ',', ';', ':', ')', oder Whitespaces.
|
|
|
|
@function clean_endpoint_path
|
|
@param {string} path - Der rohe, aus dem Text extrahierte API-Pfad.
|
|
@returns {string} Der bereinigte API-Pfad.
|
|
"""
|
|
cleaned = re.sub(r'[.,;:)\s]+$', '', path)
|
|
return cleaned.split('<')[0]
|
|
|
|
def pair_matching_endpoints(deprecated_list, active_list):
|
|
"""
|
|
Vergleicht und matcht eine Liste deprecated Endpoints mit einer Liste neuer aktiver Endpoints.
|
|
Normalisiert die Pfade (Entfernung von Versionen wie /v1/ und Plural-S-Endungen), um logische Paare zu bilden.
|
|
|
|
@function pair_matching_endpoints
|
|
@param {string[]} deprecated_list - Liste der erkannten alten Endpoints (Format: "METHOD /path").
|
|
@param {string[]} active_list - Liste der neu angebotenen Endpoints (Format: "METHOD /path").
|
|
@returns {Array} Ein Tuple bestehend aus:
|
|
{Array<[string, string]>} paired_results: Liste von Paaren [alter_endpoint, neuer_endpoint oder "-"].
|
|
{string[]} unpaired_active: Liste aktiver Endpoints, die keinem veralteten Endpoint zugeordnet werden konnten.
|
|
"""
|
|
paired_results = []
|
|
used_active = set()
|
|
|
|
for dep in deprecated_list:
|
|
dep_method, dep_path = dep.split(" ", 1)
|
|
dep_norm = re.sub(r'/v\d+/', '/', dep_path)
|
|
dep_norm = re.sub(r'/v\d+$', '', dep_norm)
|
|
dep_norm = re.sub(r's+$', '', dep_norm)
|
|
|
|
best_match = None
|
|
for act in active_list:
|
|
if act in used_active:
|
|
continue
|
|
act_method, act_path = act.split(" ", 1)
|
|
|
|
if dep_method == act_method:
|
|
act_norm = re.sub(r'/v\d+/', '/', act_path)
|
|
act_norm = re.sub(r'/v\d+$', '', act_norm)
|
|
act_norm = re.sub(r's+$', '', act_norm)
|
|
|
|
if dep_norm == act_norm:
|
|
best_match = act
|
|
break
|
|
|
|
if best_match:
|
|
paired_results.append((dep, best_match))
|
|
used_active.add(best_match)
|
|
else:
|
|
paired_results.append((dep, "-"))
|
|
|
|
unpaired_active = [act for act in active_list if act not in used_active]
|
|
return paired_results, unpaired_active
|
|
|
|
# Analysiert den Titel und die Beschreibung eines SAP "What's New"-Eintrags block- bzw. satzweise,
|
|
# identifiziert den Ankündigungstyp und extrahiert strukturierte API-Endpoint-Paare.
|
|
# @function parse_sap_description_by_blocks
|
|
# @param {string} title - Der Titel des Eintrags (enthält i.d.R. den Servicenamen).
|
|
# @param {string} description - Der mehrzeilige Fließtext mit Detailbeschreibungen und HTTP-Endpoints.
|
|
# @param {string} raw_category - Die von SAP vorgegebene Tabellen-Kategorie (z.B. "New", "Changed").
|
|
# @param {string} version_context - Die Release-Nummer als Kontext (z.B. "2605").
|
|
# @returns {Object[]} Liste von Objekten/Rows mit strukturierten API-Matrix-Informationen.
|
|
|
|
def parse_sap_description_by_blocks(title:str, description:str, raw_category:str, version_context:str):
|
|
"""
|
|
Analysiert den Titel und die Beschreibung eines SAP "What's New"-Eintrags block- bzw. satzweise,
|
|
identifiziert den Ankündigungstyp und extrahiert strukturierte API-Endpoint-Paare.
|
|
|
|
@function parse_sap_description_by_blocks
|
|
@param {string} title - Der Titel des Eintrags (enthält i.d.R. den Servicenamen).
|
|
@param {string} description - Der mehrzeilige Fließtext mit Detailbeschreibungen und HTTP-Endpoints.
|
|
@param {string} raw_category - Die von SAP vorgegebene Tabellen-Kategorie (z.B. "New", "Changed").
|
|
@param {string} version_context - Die Release-Nummer als Kontext (z.B. "2605").
|
|
@returns {Object[]} Liste von Objekten/Rows mit strukturierten API-Matrix-Informationen.
|
|
"""
|
|
parsed_rows = []
|
|
|
|
service_match = re.match(r"^([^\(]+)", title)
|
|
service_name = service_match.group(1).strip() if service_match else title
|
|
|
|
decom_match = re.search(r"decommissioned in release (\d+)", description, re.IGNORECASE)
|
|
decom_release = decom_match.group(1) if decom_match else "-"
|
|
|
|
endpoint_pattern = r"\b(GET|POST|PATCH|DELETE|PUT)\s+(/[a-zA-Z0-9_\-\{\}/\$<>]+)"
|
|
sentences = re.split(r'(?:\.|\n|:)\s*', description)
|
|
|
|
deprecated_endpoints = []
|
|
new_endpoints = []
|
|
current_state = "generic"
|
|
|
|
for sentence in sentences:
|
|
if not sentence.strip():
|
|
continue
|
|
|
|
sentence_lower = sentence.lower()
|
|
|
|
if any(word in sentence_lower for word in ["deprecated", "decommissioned", "no longer available", "removal"]):
|
|
current_state = "deprecated"
|
|
elif any(word in sentence_lower for word in ["recommend using", "use the following", "added", "added the following"]):
|
|
current_state = "new"
|
|
|
|
found_endpoints = re.findall(endpoint_pattern, sentence, re.IGNORECASE)
|
|
for method, path in found_endpoints:
|
|
clean_method = method.upper()
|
|
full_endpoint = f"{clean_method} {clean_endpoint_path(path)}"
|
|
|
|
if current_state == "deprecated":
|
|
if full_endpoint not in deprecated_endpoints:
|
|
deprecated_endpoints.append(full_endpoint)
|
|
elif current_state == "new":
|
|
if full_endpoint not in new_endpoints:
|
|
new_endpoints.append(full_endpoint)
|
|
else:
|
|
if "deprecated" in sentence_lower or "decommissioned" in sentence_lower:
|
|
if full_endpoint not in deprecated_endpoints:
|
|
deprecated_endpoints.append(full_endpoint)
|
|
else:
|
|
if full_endpoint not in new_endpoints:
|
|
new_endpoints.append(full_endpoint)
|
|
|
|
title_lower = title.lower()
|
|
desc_lower = description.lower()
|
|
|
|
if "deprecation" in title_lower or "deprecated" in desc_lower or deprecated_endpoints:
|
|
announcement_type = "Deprecation"
|
|
elif "removal" in title_lower or "removed" in desc_lower:
|
|
announcement_type = "Removal"
|
|
elif "new" in title_lower or raw_category == "New":
|
|
announcement_type = "New"
|
|
else:
|
|
announcement_type = "Change"
|
|
|
|
paired, unpaired_active = pair_matching_endpoints(deprecated_endpoints, new_endpoints)
|
|
|
|
for old_ep, new_ep in paired:
|
|
parsed_rows.append({
|
|
"Service": service_name,
|
|
"Announcement Type": "Deprecation",
|
|
"Announcement from Version": version_context,
|
|
"API-Endpoints": old_ep,
|
|
"Decommissioning in Release": decom_release,
|
|
"New Endpoints": new_ep,
|
|
"Changes": description.replace("\n", " ").strip()
|
|
})
|
|
|
|
for active_ep in unpaired_active:
|
|
parsed_rows.append({
|
|
"Service": service_name,
|
|
"Announcement Type": "Change" if announcement_type == "Deprecation" else announcement_type,
|
|
"Announcement from Version": version_context,
|
|
"API-Endpoints": active_ep,
|
|
"Decommissioning in Release": "-",
|
|
"New Endpoints": "-",
|
|
"Changes": description.replace("\n", " ").strip()
|
|
})
|
|
|
|
if not parsed_rows:
|
|
parsed_rows.append({
|
|
"Service": service_name,
|
|
"Announcement Type": announcement_type,
|
|
"Announcement from Version": version_context,
|
|
"API-Endpoints": "-",
|
|
"Decommissioning in Release": decom_release,
|
|
"New Endpoints": "-",
|
|
"Changes": description.replace("\n", " ").strip()
|
|
})
|
|
|
|
return parsed_rows
|
|
|
|
|
|
def extract_eu20_production_date(p_text:str):
|
|
"""
|
|
Durchsucht den Fließtext eines Production-Upgrades gezielt nach Datenzentren des Typs "EU20"
|
|
und extrahiert ausschließlich das dazugehörige Datum.
|
|
|
|
@function extract_eu20_production_date
|
|
@param {string} p_text - Der rohe, oft mehrzeilige Text aus der Tabellenzelle des Production Upgrades.
|
|
@returns {string} Das isolierte Datum für EU20 oder ein Fallback-Wert ("-"), falls nichts gefunden wurde.
|
|
"""
|
|
if not p_text or p_text == "-":
|
|
return "-"
|
|
|
|
# Text in Zeilen oder Abschnitte aufteilen
|
|
lines = re.split(r'\n|\r|;', p_text)
|
|
for line in lines:
|
|
if "EU20" in line:
|
|
# Sucht nach gängigen Datumsformaten (z.B. "May 16, 2026", "16.05.2026", "2026-05-16")
|
|
date_match = re.search(r'\b(?:[A-Za-z]+ \d{1,2},? \d{4}|\d{1,2}\.\d{2}\.\d{4}|\d{4}-\d{2}-\d{2})\b', line)
|
|
if date_match:
|
|
return date_match.group(0).strip()
|
|
|
|
# Fallback: Falls das Datumsformat abweicht, entferne einfach das "Data center EU20:" Label
|
|
return re.sub(r'(?i)Data\s+centers?\s+EU20\s*:\s*', '', line).strip()
|
|
|
|
# Fallback, falls "EU20" gar nicht explizit erwähnt wird (z.B. weil es nur ein einziges Datum gibt)
|
|
date_match = re.search(r'\b(?:[A-Za-z]+ \d{1,2},? \d{4}|\d{1,2}\.\d{2}\.\d{4}|\d{4}-\d{2}-\d{2})\b', p_text)
|
|
if date_match:
|
|
return date_match.group(0).strip()
|
|
|
|
return p_text.strip()
|
|
|
|
|
|
|
|
def scrape_sap_release_calendar(driver):
|
|
"""
|
|
Navigiert zum offiziellen SAP DMC Release-Kalender und extrahiert die Release-Namen
|
|
sowie die dazugehörigen Quality- und Production-Upgrade-Termine (für EU20).
|
|
|
|
@function scrape_sap_release_calendar
|
|
@param {WebDriver} driver - Die aktive Selenium Edge WebDriver Instanz.
|
|
@returns {Object.<string, {Q_Date: string, P_Date_EU20: string}>} Ein Dictionary gematcht nach Release-Name.
|
|
"""
|
|
calendar_url = "https://help.sap.com/docs/sap-digital-manufacturing/release-schedule-and-dates/release-schedule-and-dates?locale=en-US"
|
|
print(f"\nScrape den SAP Release-Kalender für alle 2026er Versionen: {calendar_url}")
|
|
|
|
calendar_data = {}
|
|
try:
|
|
driver.get(calendar_url)
|
|
wait = WebDriverWait(driver, 15)
|
|
wait.until(EC.presence_of_element_located((By.TAG_NAME, "table")))
|
|
|
|
tables = driver.find_elements(By.TAG_NAME, "table")
|
|
for table in tables:
|
|
headers = [th.text.strip().lower() for th in table.find_elements(By.TAG_NAME, "th")]
|
|
|
|
# Prüfen, ob dies die korrekte DMC-Upgrade Tabelle ist
|
|
if any("release" in h for h in headers) and any("production" in h for h in headers):
|
|
|
|
idx_release = -1
|
|
idx_q_upgrade = -1
|
|
idx_p_upgrade = -1
|
|
|
|
for i, h in enumerate(headers):
|
|
if "release" in h:
|
|
idx_release = i
|
|
elif "quality" in h or "quarterly" in h or "q-upgrade" in h:
|
|
idx_q_upgrade = i
|
|
elif "production" in h or "p-upgrade" in h:
|
|
idx_p_upgrade = i
|
|
|
|
if idx_release != -1 and idx_q_upgrade != -1 and idx_p_upgrade != -1:
|
|
rows = table.find_elements(By.XPATH, ".//tbody/tr")
|
|
for row in rows:
|
|
cells = [td.text.strip() for td in row.find_elements(By.TAG_NAME, "td")]
|
|
|
|
if len(cells) > max(idx_release, idx_q_upgrade, idx_p_upgrade):
|
|
release_name = cells[idx_release].replace("*", "").strip()
|
|
|
|
if re.match(r"^\d{4}$", release_name):
|
|
raw_p_date = cells[idx_p_upgrade]
|
|
# HIER FILTERN WIR GEZIELT NUR EU20 HERAUS
|
|
cleaned_p_date = extract_eu20_production_date(raw_p_date)
|
|
|
|
calendar_data[release_name] = {
|
|
"Q_Date": cells[idx_q_upgrade],
|
|
"P_Date_EU20": cleaned_p_date
|
|
}
|
|
break
|
|
|
|
except Exception as e:
|
|
print(f"Warnung beim Laden des Kalenders: {e}")
|
|
|
|
if calendar_data:
|
|
print(f"Erfolgreich geladene Release-Termine (bereinigt auf EU20):")
|
|
for rel, dates in calendar_data.items():
|
|
print(f" - Release {rel}: Q-Upgrade -> {dates['Q_Date']} | P-Upgrade (EU20) -> {dates['P_Date_EU20']}")
|
|
return calendar_data
|
|
|
|
|
|
def save_all_scraped_data(full_scraped_dict:dict, calendar_data:dict, output_format:str, base_filename:str):
|
|
"""
|
|
Konsolidiert alle gescrapten Release-Informationen (Metadaten + API-Änderungen) und exportiert
|
|
diese wahlweise als JSON- oder CSV-Datei in den ./output/ Ordner.
|
|
Berücksichtigt auch zukünftige/historische Releases ohne direkte "What's New"-Einträge mittels Fallbacks.
|
|
|
|
@function save_all_scraped_data
|
|
@param {Object} full_scraped_dict - Gescrapte API-Details pro Version.
|
|
@param {Object} calendar_data - Aus dem Kalender geladene Upgrade-Termine pro Version.
|
|
@param {string} output_format - Das gewünschte Exportformat ('csv' oder 'json').
|
|
@param {string} base_filename - Der Basisname der Exportdatei (ohne Dateiendung).
|
|
@returns {void}
|
|
"""
|
|
# Bilde die Menge aller bekannten Releases (aus Kalender + tatsächlich gescrapt)
|
|
all_releases = set(calendar_data.keys()).union(set(full_scraped_dict.keys()))
|
|
|
|
script_dir = os.path.dirname(os.path.abspath(__file__))
|
|
output_dir = os.path.join(script_dir, "..", "output")
|
|
|
|
|
|
if output_format == "json":
|
|
final_filename = f"{base_filename}.json"
|
|
final_filepath = os.path.join(output_dir, final_filename)
|
|
output_data = {"releases": []}
|
|
|
|
# Sortiert nach Releasename (z.B. 2511 vor 2602, 2605...)
|
|
for release_name in sorted(all_releases):
|
|
# Hole Kalender-Daten oder setze leeren Fallback anstelle eines KeyErrors
|
|
release_info = calendar_data.get(release_name, {"Q_Date": "-", "P_Date_EU20": "-"})
|
|
whats_new_list = full_scraped_dict.get(release_name, [])
|
|
|
|
output_data["releases"].append({
|
|
"ReleaseMetadata": {
|
|
"ReleaseName": release_name,
|
|
"QualityUpgradeDate_Q": release_info["Q_Date"],
|
|
"ProductionUpgradeDate_P_EU20": release_info["P_Date_EU20"]
|
|
},
|
|
"whats-new": whats_new_list
|
|
})
|
|
|
|
with open(final_filepath, "w", encoding="utf-8") as f:
|
|
json.dump(output_data, f, indent=4, ensure_ascii=False)
|
|
print(f"\n-> JSON erfolgreich in '{final_filename}' gespeichert. ({len(all_releases)} Releases abgedeckt).")
|
|
|
|
else: # CSV Format
|
|
final_filename = f"{base_filename}.csv"
|
|
final_filepath = os.path.join(output_dir, final_filename)
|
|
rows_to_save = []
|
|
|
|
for release_name in sorted(all_releases):
|
|
release_info = calendar_data.get(release_name, {"Q_Date": "-", "P_Date_EU20": "-"})
|
|
whats_new_list = full_scraped_dict.get(release_name, [])
|
|
|
|
if whats_new_list:
|
|
for entry in whats_new_list:
|
|
entry_copy = entry.copy()
|
|
entry_copy["Quality Upgrade (Q)"] = release_info["Q_Date"]
|
|
entry_copy["Production Upgrade (P - EU20)"] = release_info["P_Date_EU20"]
|
|
rows_to_save.append(entry_copy)
|
|
else:
|
|
# Zeilen-Fallback für zukünftige Releases, für die es noch keine Whats-New Daten gibt
|
|
rows_to_save.append({
|
|
"Service": "-",
|
|
"Announcement Type": "-",
|
|
"Announcement from Version": release_name,
|
|
"Quality Upgrade (Q)": release_info["Q_Date"],
|
|
"Production Upgrade (P - EU20)": release_info["P_Date_EU20"],
|
|
"API-Endpoints": "-",
|
|
"Decommissioning in Release": "-",
|
|
"New Endpoints": "-",
|
|
"Changes": "Keine Whats-New Einträge für dieses Release gelistet."
|
|
})
|
|
|
|
df = pd.DataFrame(rows_to_save)
|
|
columns_order = [
|
|
"Service", "Announcement Type", "Announcement from Version",
|
|
"Quality Upgrade (Q)", "Production Upgrade (P - EU20)",
|
|
"API-Endpoints", "Decommissioning in Release", "New Endpoints", "Changes"
|
|
]
|
|
df = df[columns_order]
|
|
df.to_csv(final_filepath, index=False, encoding="utf-8-sig", sep=";")
|
|
print(f"\n-> CSV erfolgreich in '{final_filename}' gespeichert. ({len(df)} Gesamtzeilen).")
|
|
|
|
# Ruft die spezifische SAP "What's New"-Übersichtsseite einer Version auf, wartet auf das Rendern
|
|
# der dynamischen Tabelleninhalte und extrahiert zeilenweise die Rohdaten.
|
|
# @function scrape_sap_version
|
|
# @param {WebDriver} driver - Die aktive Selenium Edge WebDriver Instanz.
|
|
# @param {string} url - Die vollständige Ziel-URL inklusive des "Version=XXXX" Parameters.
|
|
# @returns {Array} Ein Tuple bestehend aus:
|
|
# {Object[]} version_data: Die vollständig geparsten und strukturierten Zeilen für diese Version.
|
|
# {string} version_context: Die extrahierte Versionsnummer (z.B. "2511").
|
|
|
|
def scrape_sap_version(driver, url:str):
|
|
"""
|
|
Ruft die spezifische SAP "What's New"-Übersichtsseite einer Version auf, wartet auf das Rendern
|
|
der dynamischen Tabelleninhalte und extrahiert zeilenweise die Rohdaten.
|
|
|
|
@function scrape_sap_version
|
|
@param {WebDriver} driver - Die aktive Selenium Edge WebDriver Instanz.
|
|
@param {string} url - Die vollständige Ziel-URL inklusive des "Version=XXXX" Parameters.
|
|
@returns {Array} Ein Tuple bestehend aus:
|
|
{Object[]} version_data: Die vollständig geparsten und strukturierten Zeilen für diese Version.
|
|
{string} version_context: Die extrahierte Versionsnummer (z.B. "2511").
|
|
"""
|
|
version_match = re.search(r"Version=(\d+)", url)
|
|
version_context = version_match.group(1) if version_match else "2605"
|
|
|
|
print(f"\n--- Scraping Version {version_context} ---")
|
|
print(f"Lade Webseite: {url}")
|
|
driver.get(url)
|
|
|
|
wait = WebDriverWait(driver, 25)
|
|
|
|
try:
|
|
cookie_button = driver.find_element(By.ID, "truste-consent-required")
|
|
if cookie_button.is_displayed():
|
|
cookie_button.click()
|
|
time.sleep(2)
|
|
except Exception:
|
|
pass
|
|
|
|
print("Warte auf das Rendern der Tabellenzeilen...")
|
|
wait.until(EC.presence_of_element_located((By.XPATH, "//tr[starts-with(@id, 'whats-new-')]")))
|
|
|
|
table = driver.find_element(By.ID, "whats-new")
|
|
tbody_rows = table.find_elements(By.XPATH, ".//tbody/tr[starts-with(@id, 'whats-new-')]")
|
|
|
|
version_data = []
|
|
print(f"Scrape {len(tbody_rows)} Rohdaten-Zeilen...")
|
|
for row in tbody_rows:
|
|
cells = row.find_elements(By.TAG_NAME, "td")
|
|
if len(cells) < 5:
|
|
continue
|
|
|
|
raw_category = cells[2].text.strip()
|
|
title = cells[3].text.strip()
|
|
description = cells[4].text.strip()
|
|
|
|
structured_rows = parse_sap_description_by_blocks(title, description, raw_category, version_context)
|
|
version_data.extend(structured_rows)
|
|
|
|
return version_data, version_context
|
|
|
|
|
|
|
|
def main():
|
|
"""
|
|
Hauptsteuerungsfunktion (Main-Entrypoint). Lädt Umgebungsvariablen, initialisiert den Edge-Browser
|
|
im Headless-Modus, steuert den sequentiellen Scraping-Ablauf über Kalender und Versionen hinweg
|
|
und sorgt für den finalen Datenexport sowie das sichere Schließen des Browsers.
|
|
@function main
|
|
@returns {void}
|
|
"""
|
|
output_format = os.getenv("OUTPUT_FORMAT", "csv").lower().strip()
|
|
base_filename = os.getenv("OUTPUT_FILENAME", "sap_api_matrix").strip()
|
|
|
|
# Die bereits veröffentlichten URLs, die wir aktiv abfragen wollen
|
|
urls_raw = os.getenv("URLS_TO_SCRAPE", "")
|
|
if not urls_raw:
|
|
print("[FEHLER] Keine URLs in der Variable 'URLS_TO_SCRAPE' in der .env gefunden!")
|
|
return
|
|
|
|
urls_to_scrape = [url.strip() for url in urls_raw.split(",") if url.strip()]
|
|
|
|
print("Starte Edge-Browser im Firmennetzwerk-Modus...")
|
|
edge_options = Options()
|
|
edge_options.add_argument("--headless")
|
|
edge_options.add_argument("--window-size=1920,1080")
|
|
edge_options.add_argument("--lang=en-US")
|
|
edge_options.add_argument("--ignore-certificate-errors")
|
|
edge_options.add_argument("--allow-running-insecure-content")
|
|
|
|
local_driver_path = os.path.join(os.path.dirname(__file__), "..","msedgedriver.exe")
|
|
|
|
try:
|
|
service = Service(executable_path=local_driver_path)
|
|
driver = webdriver.Edge(service=service, options=edge_options)
|
|
except Exception as err:
|
|
print(f"\n[FEHLER] Edge-Initialisierung fehlgeschlagen: {err}")
|
|
return
|
|
|
|
try:
|
|
# 1. Kalender-Daten dynamisch abfragen
|
|
calendar_data = scrape_sap_release_calendar(driver)
|
|
if not calendar_data:
|
|
print("[FEHLER] Kalender-Daten konnten nicht geladen werden.")
|
|
return
|
|
|
|
full_scraped_dict = {}
|
|
|
|
# 2. Releases aus den URLs nacheinander im Browser abarbeiten
|
|
for url in urls_to_scrape:
|
|
try:
|
|
scraped_data, version = scrape_sap_version(driver, url)
|
|
if scraped_data:
|
|
full_scraped_dict[version] = scraped_data
|
|
except Exception as e:
|
|
print(f"Fehler beim Scraping der URL ({url}): {e}")
|
|
|
|
# 3. Speichere alle Daten (inklusive leerer Fallbacks für nicht gescrapte Releases wie 2608, 2611)
|
|
save_all_scraped_data(full_scraped_dict, calendar_data, output_format, base_filename)
|
|
|
|
finally:
|
|
try:
|
|
driver.quit()
|
|
except:
|
|
pass
|
|
print("\nScraping-Prozess beendet.")
|
|
|
|
if __name__ == "__main__":
|
|
main() |