Upload files to "/"

extract.py

@@ -0,0 +1,86 @@
+import camelot
+import pandas as pd
+import re
+import csv
+
+def clean_amount(val):
+    """Removes '=' and whitespace from amount strings."""
+    if not val: return ""
+    return val.replace('=', '').strip()
+
+def extract_account(text):
+    """Extracts IBAN (SI56...) from a text block."""
+    match = re.search(r'[A-Z]{2}\d{2}\s?(?:\d{4}\s?){4}\d{1,3}', text)
+    return match.group(0).strip() if match else ""
+
+def process_bank_statement(pdf_path, output_csv):
+    # Load all pages. 'stream' flavor is ideal for this layout.
+    tables = camelot.read_pdf(pdf_path, pages='all', flavor='stream', split_text=True)
+    
+    all_transactions = []
+    current_tx = None
+    
+    # Regex to identify a date (DD.MM.YY) which anchors a new transaction
+    date_pattern = re.compile(r'\d{2}\.\d{2}\.\d{2}')
+
+    for table in tables:
+        df = table.df
+        for index, row in df.iterrows():
+            # Skip the header rows (usually contains 'Creditor' or 'value date')
+            if "Creditor" in row[0] or "value date" in str(row.iloc[-1]):
+                continue
+                
+            # Detect if this row contains a date in the last column (Column 4)
+            date_val = str(row.iloc[-1]).strip()
+            is_new_tx = bool(date_pattern.search(date_val))
+
+            if is_new_tx:
+                # If we were already building a transaction, save it before starting a new one
+                if current_tx:
+                    all_transactions.append(current_tx)
+                
+                # Initialize a new transaction object
+                current_tx = {
+                    'value date': date_pattern.search(date_val).group(0),
+                    'raw_text': row[0],
+                    'debit': clean_amount(row[1]),
+                    'credit': clean_amount(row[2])
+                }
+            else:
+                # If it's not a new date, it's a continuation of the previous transaction's text
+                if current_tx and row[0].strip():
+                    current_tx['raw_text'] += " " + row[0].strip()
+
+    # Append the last transaction processed
+    if current_tx:
+        all_transactions.append(current_tx)
+
+    # Final formatting: Split raw_text into Name, Account, and Details
+    final_data = []
+    for tx in all_transactions:
+        text = tx['raw_text']
+        account = extract_account(text)
+        
+        # Simple split logic: First part is Name, rest is Details (excluding account)
+        # Note: This logic can be refined based on specific name lengths
+        details = text.replace(account, "").strip()
+        parts = details.split("  ", 1) # Try to find a larger gap
+        name = parts[0].strip()
+        payment_details = parts[1].strip() if len(parts) > 1 else details
+        
+        final_data.append({
+            "value date": tx['value date'],
+            "Creditor / Debtor": name,
+            "account": account,
+            "payment details": payment_details,
+            "debit": tx['debit'],
+            "credit": tx['credit']
+        })
+
+    # Export to CSV with semicolon delimiter
+    output_df = pd.DataFrame(final_data)
+    output_df.to_csv(output_csv, index=False, sep=';', encoding='utf-8-sig', quoting=csv.QUOTE_ALL)
+    print(f"Successfully exported {len(final_data)} transactions to {output_csv}")
+
+# Execution
+process_bank_statement("MSDataReport (1).pdf", "Processed_Transactions.csv")

extract_bank_fixed.py

@@ -0,0 +1,200 @@
+import camelot
+import pandas as pd
+import re
+from pathlib import Path
+
+
+def extract_bank_data(file_path: str, output_csv: str = "bank_export_fixed.csv"):
+    print(f"Reading {file_path}… processing all pages.")
+
+    # Tune these if needed:
+    # - edge_tol: how aggressively Camelot merges columns
+    # - row_tol:  how aggressively it merges rows
+    tables = camelot.read_pdf(
+        file_path,
+        pages="1-end",
+        flavor="stream",
+        strip_text="\n",
+    )
+
+    print(f"Found {len(tables)} tables")
+
+    final_transactions = []
+    current_tx = None
+
+    # Regex patterns (adapt as needed for your bank)
+    iban_pattern = r"SI56\s?\d{4}\s?\d{4}\s?\d{4}\s?\d{3}"
+    amount_pattern = r"=(\d+[\.,]\d{2})"
+    date_pattern = r"(\d{2}\.\d{2}\.\d{2})"
+
+    # Helper: detect which column holds debit vs credit by position
+    # You might want to print some sample rows to confirm indexes.
+    def get_debit_credit_from_row(cells):
+        """
+        Decide whether the amount we found is debit or credit based on
+        which column index contains '='.
+        You can adapt the column index test to your PDF.
+        """
+        # Example strategy:
+        # - assume columns are like: [desc, extra, date, debit_col, credit_col]
+        # - if '=' appears first in column <= 3 => debit, else credit
+
+        eq_indices = [i for i, c in enumerate(cells) if "=" in c]
+        if not eq_indices:
+            return None, None, None  # (amount, debit, credit)
+
+        # Use the first '=' we find
+        idx = eq_indices[0]
+        m = re.search(amount_pattern, cells[idx])
+        if not m:
+            return None, None, None
+
+        raw_amt = m.group(1).replace(",", ".").strip()  # normalize to dot
+
+        # Heuristic by index: tweak these numbers based on actual layout
+        # Example: if index <= 2 -> debit, else credit
+        is_debit = idx <= 2
+
+        debit = raw_amt if is_debit else ""
+        credit = "" if is_debit else raw_amt
+        return raw_amt, debit, credit
+
+    for t_index, table in enumerate(tables, start=1):
+        df = table.df
+
+        # Optionally skip header rows on page 1
+        # You can detect header by known header text, e.g. "Vrednost" or similar
+        for _, row in df.iterrows():
+            cells = [str(c).strip() for c in row]
+            row_text = " ".join(c for c in cells if c)
+
+            if not row_text:
+                continue
+
+            # --- 1. New transaction row? ---
+            date_match = re.search(date_pattern, row_text)
+            raw_amt, debit, credit = get_debit_credit_from_row(cells)
+
+            if raw_amt is not None and date_match:
+                # Save any previous transaction before starting a new one
+                if current_tx:
+                    final_transactions.append(current_tx)
+
+                value_date = date_match.group(1)
+
+                # Creditor/Debtor is usually leftmost column(s).
+                # Here we take cell 0 as the primary label.
+                creditor_debtor = cells[0]
+
+                current_tx = {
+                    "value date": value_date,
+                    "Creditor / Debtor": creditor_debtor,
+                    "account": "",
+                    "payment details": "",
+                    "debit": debit,
+                    "credit": credit,
+                    "is_collecting_details": False,
+                }
+
+                # IBAN possibly already in the same row
+                for c in cells:
+                    iban_search = re.search(iban_pattern, c)
+                    if iban_search:
+                        current_tx["account"] = iban_search.group(0)
+                        # Everything after IBAN is usually detail text
+                        rem = c[iban_search.end() :].strip()
+                        rem = re.sub(r"[A-Z]{8,11}", "", rem).strip()
+                        if rem:
+                            current_tx["payment details"] = (
+                                current_tx["payment details"] + " " + rem
+                            ).strip()
+                        current_tx["is_collecting_details"] = True
+                        break
+
+                continue  # go to next row
+
+            # --- 2. Continuation row of last transaction ---
+            if current_tx:
+                text_line = cells[0].strip()
+                if not text_line:
+                    continue
+
+                # IBAN?
+                iban_search = re.search(iban_pattern, text_line)
+                if iban_search:
+                    current_tx["account"] = iban_search.group(0)
+                    current_tx["is_collecting_details"] = True
+                    remaining = re.sub(iban_pattern, "", text_line).strip()
+                    remaining = re.sub(r"[A-Z]{8,11}", "", remaining).strip()
+                    if remaining:
+                        current_tx["payment details"] = (
+                            current_tx["payment details"] + " " + remaining
+                        ).strip()
+                    continue
+
+                # Heuristic: address lines to skip from 'name' and usually not details
+                is_address = any(
+                    kw in text_line.upper()
+                    for kw in ["CESTA", "ULICA", "TRG", " LJUBLJANA", " LOGATEC"]
+                )
+
+                # If we are already collecting payment details or see typical reference markers
+                if (
+                    current_tx["is_collecting_details"]
+                    or "SI00" in text_line
+                    or "SI12" in text_line
+                ):
+                    clean_text = re.sub(r"[A-Z]{8,11}", "", text_line).strip()
+                    if clean_text:
+                        current_tx["payment details"] = (
+                            current_tx["payment details"] + " " + clean_text
+                        ).strip()
+                elif not is_address and not current_tx["account"]:
+                    # Continuation of very short creditor/debtor name
+                    if len(current_tx["Creditor / Debtor"]) < 20:
+                        current_tx["Creditor / Debtor"] = (
+                            current_tx["Creditor / Debtor"] + " " + text_line
+                        ).strip()
+
+    # Append last transaction if present
+    if current_tx:
+        final_transactions.append(current_tx)
+
+    # --- 3. To DataFrame and CSV ---
+    df_final = pd.DataFrame(final_transactions)
+
+    # Drop internal flag
+    if "is_collecting_details" in df_final.columns:
+        df_final = df_final.drop(columns=["is_collecting_details"])
+
+    # Clean text slightly
+    if "payment details" in df_final.columns:
+        df_final["payment details"] = df_final["payment details"].str.replace(
+            r"[A-Z]{8,11}", "", regex=True
+        )
+        df_final["payment details"] = (
+            df_final["payment details"]
+            .str.replace(r"\s+", " ", regex=True)
+            .str.strip()
+        )
+
+    # Keep columns in desired order
+    desired_cols = [
+        "value date",
+        "Creditor / Debtor",
+        "account",
+        "payment details",
+        "debit",
+        "credit",
+    ]
+    df_final = df_final.reindex(columns=desired_cols)
+
+    # Export as ;‑delimited CSV
+    df_final.to_csv(output_csv, sep=";", index=False, encoding="utf-8-sig")
+    print(f"Success! Exported {len(df_final)} transactions to {output_csv}")
+
+
+if __name__ == "__main__":
+    pdf_path = "MSDataReport (1).pdf"  # adjust if needed
+    out_csv = "bank_export_fixed.csv"
+    extract_bank_data(pdf_path, out_csv)

extract_bank_github.py

@@ -0,0 +1,185 @@
+import os
+import re
+import camelot
+import pandas as pd
+
+# --- CONFIGURATION ---
+
+PDF_PATH = r"MSDataReport (1).pdf"  # PDF filename in the current directory
+OUTPUT_CSV = r"transactions.csv"    # Output semicolon-delimited CSV
+PAGES = "1-end"                     # All pages; change if needed
+
+# These are the logical columns we want in the final CSV
+TARGET_COLUMNS = [
+    "value date",
+    "Creditor / Debtor",
+    "account",
+    "payment details",
+    "debit",
+    "credit",
+]
+
+# If the header row in the PDF is slightly different, you can map it here.
+# Keys are patterns you expect in the PDF header; values are our normalized names.
+HEADER_MAP = {
+    # left side is a regex pattern (case-insensitive)
+    r"value\s*date": "value date",
+    r"creditor\s*/\s*debtor": "Creditor / Debtor",
+    r"account": "account",
+    r"payment\s*details": "payment details",
+    r"debit": "debit",
+    r"credit": "credit",
+}
+
+
+def normalize_header(col_name: str) -> str:
+    """
+    Normalize a raw column header from the PDF to one of TARGET_COLUMNS (if possible).
+    Uses HEADER_MAP regex patterns.
+    """
+    if not isinstance(col_name, str):
+        return col_name
+
+    name = col_name.strip().lower()
+    for pattern, target in HEADER_MAP.items():
+        if re.search(pattern, name, flags=re.IGNORECASE):
+            return target
+    return col_name.strip()
+
+
+def is_header_row(row_values):
+    """
+    Heuristically determine if a row in the table is the header row:
+    - If it contains at least two of the expected header keywords.
+    """
+    header_text = " ".join(str(x) for x in row_values if isinstance(x, str)).lower()
+    hits = 0
+    for pattern in HEADER_MAP.keys():
+        if re.search(pattern, header_text, re.IGNORECASE):
+            hits += 1
+    return hits >= 2
+
+
+def clean_amount(value: str) -> str:
+    """
+    Clean debit/credit amount strings:
+    - Strip spaces
+    - Convert localized formats to plain decimal (e.g., "1 234,56-" -> "-1234.56")
+    """
+    if not isinstance(value, str):
+        return ""
+
+    v = value.strip()
+    if not v:
+        return ""
+
+    # Move trailing minus to the front for easier numeric parsing later
+    if v.endswith("-"):
+        v = "-" + v[:-1].strip()
+
+    # Remove thousand separators (spaces or dots depending on locale)
+    v = v.replace(" ", "")
+    # Common European formatting: "1.234,56"
+    # Replace thousand '.' with nothing, and decimal ',' with '.'
+    if "," in v and "." in v and v.rfind(",") > v.rfind("."):
+        v = v.replace(".", "").replace(",", ".")
+    else:
+        # If only ',' present, assume it's decimal separator
+        if "," in v and "." not in v:
+            v = v.replace(",", ".")
+
+    # Final sanity strip
+    return v
+
+
+def main():
+    if not os.path.exists(PDF_PATH):
+        raise FileNotFoundError(f"PDF file not found: {PDF_PATH}")
+
+    print(f"Reading tables from {PDF_PATH} on pages: {PAGES} ...")
+    # flavor="lattice" works best when there are visible cell borders
+    # If results are poor, try flavor="stream"
+    tables = camelot.read_pdf(
+        PDF_PATH,
+        pages=PAGES,
+        flavor="lattice",
+        strip_text="\n",
+    )
+
+    print(f"Found {len(tables)} table(s). Processing...")
+
+    all_rows = []
+
+    for idx, table in enumerate(tables):
+        df = table.df.copy()
+        print(f"Processing table {idx+1}/{len(tables)} with shape {df.shape}")
+
+        if df.empty:
+            continue
+
+        # Detect header row (row index where header is located)
+        header_row_idx = None
+        for i in range(min(5, len(df))):  # search first few rows
+            if is_header_row(df.iloc[i].tolist()):
+                header_row_idx = i
+                break
+
+        if header_row_idx is None:
+            print(f"  Warning: No header row detected in table {idx+1}; skipping this table.")
+            continue
+
+        # Use that row as header, drop all rows up to that row
+        df.columns = [normalize_header(c) for c in df.iloc[header_row_idx]]
+        df = df.iloc[header_row_idx + 1:].reset_index(drop=True)
+
+        # Keep only columns we care about (if they exist)
+        # But first, ensure uniqueness of column names
+        df = df.loc[:, ~df.columns.duplicated()]
+
+        present_cols = [c for c in df.columns if c in TARGET_COLUMNS]
+        missing_cols = [c for c in TARGET_COLUMNS if c not in present_cols]
+
+        if missing_cols:
+            print(f"  Note: in table {idx+1}, these target columns are missing: {missing_cols}")
+
+        # Reindex with our target columns, missing will be filled with empty strings
+        df = df.reindex(columns=TARGET_COLUMNS)
+
+        # Drop rows that are completely empty
+        df = df.replace(r"^\s*$", pd.NA, regex=True)
+        df = df.dropna(how="all")
+
+        # Clean debit/credit formats
+        if "debit" in df.columns:
+            df["debit"] = df["debit"].apply(lambda x: clean_amount(str(x)) if pd.notna(x) else "")
+        if "credit" in df.columns:
+            df["credit"] = df["credit"].apply(lambda x: clean_amount(str(x)) if pd.notna(x) else "")
+
+        # Append to master list, preserving the order Camelot returns tables (which is page order)
+        all_rows.extend(df.to_dict(orient="records"))
+
+    if not all_rows:
+        raise RuntimeError("No data rows extracted. Check that the PDF has tables and try flavor='stream'.")
+
+    # Convert to DataFrame and sort by "value date" to enforce chronological order
+    out_df = pd.DataFrame(all_rows, columns=TARGET_COLUMNS)
+
+    # Try parsing the date to sort chronologically; if it fails, preserve original order
+    try:
+        out_df["__parsed_date"] = pd.to_datetime(out_df["value date"], dayfirst=True, errors="coerce")
+        if out_df["__parsed_date"].notna().any():
+            out_df = out_df.sort_values(["__parsed_date", "value date"]).reset_index(drop=True)
+        out_df = out_df.drop(columns=["__parsed_date"])
+    except Exception as e:
+        print(f"Warning: could not parse dates for sorting: {e}")
+        # leave out_df as-is
+
+    # Export as semicolon-delimited CSV with UTF-8 encoding
+    out_df.to_csv(OUTPUT_CSV, sep=";", index=False, encoding="utf-8")
+
+    print(f"Done. Extracted {len(out_df)} rows into {OUTPUT_CSV}")
+    print("Columns:", list(out_df.columns))
+
+
+if __name__ == "__main__":
+    main()

extract_bank_newest.py

@@ -0,0 +1,104 @@
+import camelot
+import pandas as pd
+import re
+
+def extract_bank_data(file_path):
+    print(f"Reading {file_path}... shifting Payment Details parsing one row lower.")
+    
+    # We use 'stream' flavor for the NLB report layout
+    tables = camelot.read_pdf(file_path, pages='1-end', flavor='stream')
+    
+    final_transactions = []
+    current_tx = None
+    
+    # Patterns
+    iban_pattern = r'SI56\s?\d{4}\s?\d{4}\s?\d{4}\s?\d{3}'
+    amount_pattern = r'=(\d+[\.,]\d{2})'
+    date_pattern = r'(\d{2}\.\d{2}\.\d{2})'
+    bic_pattern = r'\b[A-Z0-9]{8,11}\b'
+
+    for table in tables:
+        df = table.df
+        for _, row in df.iterrows():
+            cells = [str(c).strip() for c in row]
+            row_text = " ".join(cells)
+            
+            amount_match = re.search(amount_pattern, row_text)
+            date_match = re.search(date_pattern, row_text)
+            
+            # TRIGGER: New transaction starts
+            if amount_match and date_match:
+                if current_tx:
+                    final_transactions.append(current_tx)
+                
+                amt_val = amount_match.group(1)
+                
+                # Debit/Credit alignment logic
+                is_credit = False
+                for i, c in enumerate(cells):
+                    if '=' in c and i >= 2:
+                        is_credit = True
+                
+                current_tx = {
+                    "value date": date_match.group(1),
+                    "Creditor / Debtor": cells[0].split('\n')[0].strip(),
+                    "account": "",
+                    "payment details": "",
+                    "debit": "" if is_credit else amt_val,
+                    "credit": amt_val if is_credit else "",
+                    "skip_rows": 0  # Counter to shift parsing lower
+                }
+            
+            elif current_tx:
+                text_line = cells[0]
+                if not text_line: continue
+
+                # 1. Identify IBAN
+                iban_search = re.search(iban_pattern, text_line)
+                if iban_search:
+                    current_tx["account"] = iban_search.group(0).replace(" ", "")
+                    # When IBAN is found, the line below it is usually BIC. 
+                    # We set skip_rows to 1 to skip that BIC line.
+                    current_tx["skip_rows"] = 1
+                    continue
+
+                # 2. Skip the "BIC" line (one row lower logic)
+                if current_tx["skip_rows"] > 0:
+                    current_tx["skip_rows"] -= 1
+                    continue
+
+                # 3. Capture Payment Details (after skipping)
+                # Filter out obvious address noise
+                is_address = any(k in text_line.upper() for k in ["CESTA", "ULICA", "TRG", "LJUBLJANA"]) or re.search(r'\b\d{4}\b', text_line)
+                
+                if not is_address and len(text_line) > 2:
+                    # Scrub labels and internal bank noise
+                    clean_text = re.sub(r'^(purpose|payment details|reference)', '', text_line, flags=re.I).strip()
+                    clean_text = re.sub(r'\b[CD]R\d+\b|\bNRC\b', '', clean_text).strip()
+                    
+                    if clean_text:
+                        # Append to details
+                        if not current_tx["payment details"]:
+                            current_tx["payment details"] = clean_text
+                        else:
+                            current_tx["payment details"] += " " + clean_text
+
+    if current_tx:
+        final_transactions.append(current_tx)
+
+    # Export
+    df_final = pd.DataFrame(final_transactions).drop(columns=['skip_rows'])
+    cols = ["value date", "Creditor / Debtor", "account", "payment details", "debit", "credit"]
+    df_final = df_final[cols]
+    
+    # Final cleanup
+    for col in df_final.columns:
+        df_final[col] = df_final[col].astype(str).str.replace(r'\s+', ' ', regex=True).str.strip()
+        df_final[col] = df_final[col].replace('nan', '')
+
+    output_file = "bank_export_final_fixed.csv"
+    df_final.to_csv(output_file, sep=';', index=False, encoding='utf-8-sig')
+    print(f"Success! {len(df_final)} transactions exported.")
+
+if __name__ == "__main__":
+    extract_bank_data("MSDataReport (1).pdf")