andrey.cunha/trm_rem_saggezza
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

att

Browse Source
This commit is contained in:
andrey.cunha
2026-05-22 14:24:01 -03:00
parent b57fae393e
commit bbb9a16986
7 changed files with 770 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
installments_reader.py
+291 -182
View File
@@ -3,6 +3,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
import json
import os
import re
import threading
import time
from dataclasses import dataclass
from datetime import date, timedelta
@@ -14,10 +15,27 @@ import requests
TOKENS_URL = "https://api.grupoginseng.com.br/api/tokens"
STORES_URL = "https://api-extranet.grupoboticario.digital/api/person-logged/stores"
INSTALLMENTS_URL = (
"https://bff-credit-container-portal-apigw.produto-financeiro.grupoboticario.digital/"
"https://bff-credit-container-portal-apigw.prd.produto-financeiro.app.grupoboticario.com.br/"
"v1/franchisee/installments"
)
_RATE_LIMIT_LOCK = threading.Lock()
_LAST_INSTALLMENTS_REQUEST_TS = 0.0
def _set_global_backoff(wait_s: float) -> None:
"""After a 429, push the shared timestamp forward so ALL threads pause."""
global _LAST_INSTALLMENTS_REQUEST_TS
min_interval_ms_raw = os.getenv("INSTALLMENTS_MIN_INTERVAL_MS", "0").strip()
try:
min_interval_s = int(min_interval_ms_raw or "0") / 1000.0
except Exception:
min_interval_s = 0.0
with _RATE_LIMIT_LOCK:
target = time.monotonic() + wait_s - min_interval_s
if target > _LAST_INSTALLMENTS_REQUEST_TS:
_LAST_INSTALLMENTS_REQUEST_TS = target
def _jwt_payload(jwt_token: str) -> Dict[str, Any]:
parts = jwt_token.split(".")
@@ -65,12 +83,40 @@ class Auth:
self.cache = TokenCache()
def _apply_installments_pacing() -> None:
min_interval_ms_raw = os.getenv("INSTALLMENTS_MIN_INTERVAL_MS", "0").strip()
try:
min_interval_ms = int(min_interval_ms_raw or "0")
except Exception:
min_interval_ms = 0
if min_interval_ms <= 0:
return
min_interval_s = float(min_interval_ms) / 1000.0
global _LAST_INSTALLMENTS_REQUEST_TS
while True:
with _RATE_LIMIT_LOCK:
now = time.monotonic()
wait_s = (_LAST_INSTALLMENTS_REQUEST_TS + min_interval_s) - now
if wait_s <= 0:
_LAST_INSTALLMENTS_REQUEST_TS = now
return
time.sleep(min(wait_s, 1.0))
def _headers(auth: Auth, cookie_header: Optional[str]) -> Dict[str, str]:
h = {
"Authorization": auth.get_bearer(),
"Accept": "application/json, text/plain, */*",
"Accept": "*/*",
"Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8,en-US;q=0.7",
"Cache-Control": "no-cache",
"Pragma": "no-cache",
"Content-Type": "application/json",
"Origin": "https://extranet.grupoboticario.com.br",
"Referer": "https://extranet.grupoboticario.com.br/",
"Sec-Fetch-Dest": "empty",
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "cross-site",
"User-Agent": "Mozilla/5.0",
}
if cookie_header:
@@ -88,8 +134,11 @@ def get_installments_page(
page: int,
installment_group_code: Optional[str] = None,
cookie_header: Optional[str] = None,
page_size: Optional[int] = None,
) -> Dict[str, Any]:
params: Dict[str, Any] = {"page": page}
if page_size and page_size > 0:
params["limit"] = page_size
if installment_group_code:
params["installmentGroupCode"] = installment_group_code
else:
@@ -105,6 +154,7 @@ def get_installments_page(
max_attempts = 8
transient_status = {429, 500, 502, 503, 504}
for attempt in range(max_attempts):
_apply_installments_pacing()
r = session.get(
INSTALLMENTS_URL,
headers=_headers(auth, cookie_header=cookie_header),
@@ -124,6 +174,21 @@ def get_installments_page(
break
if r.status_code in transient_status:
wait_s = min(30, 2 ** min(5, attempt))
if r.status_code == 429:
min_429_wait_raw = os.getenv("INSTALLMENTS_429_MIN_WAIT_SEC", "3").strip()
try:
min_429_wait = float(min_429_wait_raw or "3")
except Exception:
min_429_wait = 3.0
retry_after = r.headers.get("Retry-After") or r.headers.get("retry-after")
if retry_after:
try:
min_429_wait = max(min_429_wait, float(retry_after))
print(f"[info] Retry-After da API: {retry_after}s")
except Exception:
pass
wait_s = max(wait_s, min_429_wait)
_set_global_backoff(wait_s)
print(
f"[warn] erro temporario {r.status_code} no installments "
f"(tentativa {attempt + 1}/{max_attempts}), aguardando {wait_s}s..."
@@ -142,6 +207,13 @@ def get_installments_page(
f"url={INSTALLMENTS_URL}?{urlencode(params)} body={r.text[:500]}"
)
if r.status_code in transient_status:
if r.status_code == 429:
recovery_s = float(os.getenv("THROTTLE_RECOVERY_PAUSE_SEC", "900"))
print(
f"[throttle-recovery] 429 persistente apos {max_attempts} tentativas. "
f"Aplicando cooldown global de {int(recovery_s)}s para proxima loja..."
)
_set_global_backoff(recovery_s)
raise RuntimeError(
f"Falha temporaria persistente ({r.status_code}) no installments apos {max_attempts} tentativas. "
f"url={INSTALLMENTS_URL}?{urlencode(params)} body={r.text[:500]}"
@@ -151,14 +223,31 @@ def get_installments_page(
f"400 Bad Request em installments. "
f"url={INSTALLMENTS_URL}?{urlencode(params)} body={r.text[:500]}"
)
if r.status_code == 412:
raise RuntimeError(
"412 Precondition Failed em installments. "
"A API recusou a pre-condicao da requisicao (normalmente cabecalhos/sessao). "
f"url={INSTALLMENTS_URL}?{urlencode(params)} body={r.text[:500]}"
)
r.raise_for_status()
return r.json()
def get_store_codes(session: requests.Session, auth: Auth, cookie_header: Optional[str]) -> list[int]:
def get_store_codes(
session: requests.Session,
auth: Auth,
cookie_header: Optional[str],
allowed_channels: Optional[set[str]] = None,
) -> list[int]:
r = session.get(STORES_URL, headers=_headers(auth, cookie_header=cookie_header), timeout=30)
r.raise_for_status()
data = r.json().get("data") or []
if allowed_channels:
data = [
x
for x in data
if str(x.get("channel") or "").strip().casefold() in allowed_channels
]
out = sorted({int(x.get("code")) for x in data if x.get("code") is not None})
return out
@@ -453,21 +542,37 @@ WHERE DocPedidoId = ? AND InstallmentCode NOT IN ({placeholders})
def main() -> None:
today = date.today()
last_days_env = os.getenv("LAST_N_DAYS", "5").strip()
last_n_days = int(last_days_env) if last_days_env else None
rolling_start = (today - timedelta(days=last_n_days)).isoformat() if last_n_days is not None else None
default_start = date(today.year, 1, 1).isoformat()
default_end = today.isoformat()
start_date_env = os.getenv("START_INSTALLMENT_CHANGE_DATE")
start_date_fixed = (start_date_env or "").strip() or None
last_days_env = os.getenv("LAST_N_DAYS", "5").strip()
last_n_days = int(last_days_env) if last_days_env else None
rolling_start = None
if (not start_date_fixed) and (last_n_days is not None):
rolling_start = (today - timedelta(days=last_n_days)).isoformat()
chunk_days_env = os.getenv("CHUNK_DAYS", "0").strip()
chunk_days = int(chunk_days_env) if chunk_days_env else 0
if chunk_days < 0:
chunk_days = 0
default_start = date(today.year, 1, 1).isoformat()
default_end = today.isoformat()
end_date = os.getenv("END_INSTALLMENT_CHANGE_DATE", default_end)
installment_change = os.getenv("INSTALLMENT_CHANGE", "CRIACAO").strip() or "CRIACAO"
first_page = int(os.getenv("PAGE_START", "1"))
max_pages_per_query = int(os.getenv("MAX_PAGES_PER_QUERY", "10000"))
page_size_env = os.getenv("INSTALLMENTS_PAGE_SIZE", "").strip()
page_size = int(page_size_env) if page_size_env else None
flush_every_pages = int(os.getenv("FLUSH_EVERY_PAGES", "50"))
# Padrao organizado: uma loja por vez (logs nao ficam intercalados).
store_workers = int(os.getenv("STORE_WORKERS", "1"))
group_workers = int(os.getenv("GROUP_WORKERS", "4"))
cookie_header = os.getenv("EXTRANET_COOKIE")
store_channel_env = os.getenv("STORE_CHANNEL", "VD").strip()
allowed_channels = {
c.strip().casefold()
for c in re.split(r"[,\s;]+", store_channel_env)
if c.strip()
}
if "all" in allowed_channels or "*" in allowed_channels:
allowed_channels = set()
target_mediator_env = os.getenv("TARGET_MEDIATOR_CODE", "").strip()
target_mediator = int(target_mediator_env) if target_mediator_env else None
resume_from_env = os.getenv("RESUME_FROM_MEDIATOR_CODE", "").strip()
@@ -494,7 +599,15 @@ def main() -> None:
session.trust_env = False
auth = Auth(session)
stores = get_store_codes(session, auth, cookie_header)
stores = get_store_codes(
session,
auth,
cookie_header,
allowed_channels=allowed_channels or None,
)
if allowed_channels:
channels_txt = ",".join(sorted(c.upper() for c in allowed_channels))
print(f"[info] filtro de canal ativo: STORE_CHANNEL={channels_txt} (lojas={len(stores)})")
if target_mediator is not None:
stores = [target_mediator]
print(f"[info] consulta focada na loja {target_mediator}")
@@ -509,8 +622,10 @@ def main() -> None:
print(f"[info] incremental ativo com watermark em {watermark_file}")
else:
print("[info] incremental desativado")
if last_n_days is not None:
if rolling_start is not None and last_n_days is not None:
print(f"[info] janela movel ativa: ultimos {last_n_days} dias ({rolling_start}..{end_date})")
if chunk_days > 0:
print(f"[info] fatiamento de periodo ativo: CHUNK_DAYS={chunk_days}")
authorized: list[int] = []
unauthorized: list[int] = []
@@ -531,173 +646,158 @@ def main() -> None:
local_session = requests.Session()
local_session.trust_env = False
local_auth = Auth(local_session)
store_bearer = local_auth.get_bearer()
try:
print(
f"[consulta] loja={mediator_code} periodo={store_start_date}..{end_date} pagina_inicial={first_page}"
)
page = first_page
all_installments: List[Dict[str, Any]] = []
store_start_obj = _parse_iso_date(store_start_date)
store_end_obj = _parse_iso_date(end_date)
if not store_start_obj or not store_end_obj:
raise RuntimeError(
f"Periodo invalido para consulta. start={store_start_date} end={end_date}"
)
if store_start_obj > store_end_obj:
raise RuntimeError(
f"Periodo invalido para consulta. start={store_start_date} end={end_date}"
)
windows: List[tuple[str, str]] = []
if chunk_days > 0:
cursor = store_start_obj
step_days = max(1, chunk_days)
while cursor <= store_end_obj:
win_end = min(store_end_obj, cursor + timedelta(days=step_days - 1))
windows.append((cursor.isoformat(), win_end.isoformat()))
cursor = win_end + timedelta(days=1)
else:
windows = [(store_start_obj.isoformat(), store_end_obj.isoformat())]
all_group_codes_seen: set = set()
total_api = 0
total_pages = 1
while True:
try:
body = get_installments_page(
session=local_session,
auth=local_auth,
start_date=store_start_date,
end_date=end_date,
installment_change=installment_change,
mediator_code=mediator_code,
page=page,
cookie_header=cookie_header,
)
except Exception as e:
msg = str(e)
if "400 Bad Request" in msg and installment_change:
pages_fetched_total = 0
total_sql_pedidos = 0
total_sql_parcelas = 0
def _flush(buffer: List[Dict[str, Any]], label: str) -> None:
"""Agrupa itens do scan por installmentGroupCode e persiste no banco."""
nonlocal total_sql_pedidos, total_sql_parcelas
if not buffer:
return
groups: Dict[str, List[Dict[str, Any]]] = {}
for item in _dedupe_installments(buffer):
gc = str(item.get("installmentGroupCode") or "").strip()
if not gc or gc in all_group_codes_seen:
continue
groups.setdefault(gc, []).append(item)
if not groups:
return
all_group_codes_seen.update(groups.keys())
if log_group_codes:
for gc in groups:
print(f"[grupo] loja={mediator_code} {label} installmentGroupCode={gc}")
print(f"[flush] loja={mediator_code} {label} grupos_novos={len(groups)}")
if write_sql:
sql_rows: List[Dict[str, Any]] = []
for gc, items in groups.items():
sql_rows.append({
"mediatorCode": mediator_code,
"installmentGroupCode": gc,
"rawResponse": {"data": {"installments": items}},
"installments": items,
})
if sql_rows:
stats = upsert_doc_pedidos_sqlserver(sql_rows, sql_conn, mediator_code_log=mediator_code)
total_sql_pedidos += stats.get("pedidos", 0)
total_sql_parcelas += stats.get("parcelas", 0)
print(
f"[fallback] loja={mediator_code} 400 com installmentChange={installment_change}; "
"tentando sem installmentChange"
f"[sql-flush] loja={mediator_code} {label} "
f"pedidos_upsert={stats.get('pedidos')} parcelas_upsert={stats.get('parcelas')}"
)
for window_idx, (window_start, window_end) in enumerate(windows, start=1):
print(
f"[consulta-janela] loja={mediator_code} janela={window_idx}/{len(windows)} "
f"periodo={window_start}..{window_end}"
)
page = first_page
window_total = 0
window_total_pages = 1
window_pages_fetched = 0
flush_buffer: List[Dict[str, Any]] = []
while True:
try:
body = get_installments_page(
session=local_session,
auth=local_auth,
start_date=store_start_date,
end_date=end_date,
installment_change=None,
start_date=window_start,
end_date=window_end,
installment_change=installment_change,
mediator_code=mediator_code,
page=page,
cookie_header=cookie_header,
page_size=page_size,
)
else:
raise
installments_page = (((body.get("data") or {}).get("installments")) or [])
all_installments.extend(installments_page)
pagination = ((body.get("data") or {}).get("pagination") or {})
total_api = int(pagination.get("total") or len(all_installments))
limit = int(pagination.get("limit") or len(installments_page) or 1)
total_pages = max(1, (total_api + limit - 1) // limit) if total_api else page
print(
f"[pagina-loja] loja={mediator_code} pagina={page}/{total_pages} "
f"itens_pagina={len(installments_page)}"
)
if not installments_page:
break
if page >= total_pages:
break
if page - first_page + 1 >= max_pages_per_query:
except Exception as e:
msg = str(e)
if "400 Bad Request" in msg and installment_change:
print(
f"[fallback] loja={mediator_code} 400 com installmentChange={installment_change}; "
"tentando sem installmentChange"
)
body = get_installments_page(
session=local_session,
auth=local_auth,
start_date=window_start,
end_date=window_end,
installment_change=None,
mediator_code=mediator_code,
page=page,
cookie_header=cookie_header,
page_size=page_size,
)
else:
raise
installments_page = (((body.get("data") or {}).get("installments")) or [])
flush_buffer.extend(installments_page)
pagination = ((body.get("data") or {}).get("pagination") or {})
window_total = int(pagination.get("total") or 0)
limit = int(pagination.get("limit") or len(installments_page) or 1)
window_total_pages = max(1, (window_total + limit - 1) // limit) if window_total else page
window_pages_fetched += 1
print(
f"[stop] loja={mediator_code} limite MAX_PAGES_PER_QUERY={max_pages_per_query} atingido"
f"[pagina-loja] loja={mediator_code} janela={window_idx}/{len(windows)} "
f"pagina={page}/{window_total_pages} itens_pagina={len(installments_page)}"
)
break
page += 1
if flush_every_pages > 0 and window_pages_fetched % flush_every_pages == 0:
_flush(flush_buffer, f"janela={window_idx}/{len(windows)} pagina={page}")
flush_buffer = []
if not installments_page:
break
if page >= window_total_pages:
break
if page - first_page + 1 >= max_pages_per_query:
print(
f"[stop] loja={mediator_code} janela={window_idx}/{len(windows)} "
f"limite MAX_PAGES_PER_QUERY={max_pages_per_query} atingido"
)
break
page += 1
# Flush do restante da janela que não atingiu o threshold
_flush(flush_buffer, f"janela={window_idx}/{len(windows)}")
total_api += window_total
pages_fetched_total += window_pages_fetched
print(
f"[resultado-janela] loja={mediator_code} janela={window_idx}/{len(windows)} "
f"grupos_acumulados={len(all_group_codes_seen)}"
)
installments = _dedupe_installments(all_installments)
group_codes = sorted(
{
str(item.get("installmentGroupCode")).strip()
for item in installments
if item.get("installmentGroupCode") is not None
and str(item.get("installmentGroupCode")).strip()
}
)
print(
f"[resultado-loja] loja={mediator_code} pedidos_encontrados={total_api} "
f"itens_total_agregados={len(installments)} grupos_unicos={len(group_codes)}"
f"grupos_unicos={len(all_group_codes_seen)}"
)
if log_group_codes:
for gc in group_codes:
print(f"[grupo] loja={mediator_code} installmentGroupCode={gc}")
pedidos: Dict[str, Any] = {}
def fetch_group(group_code: str) -> Dict[str, Any]:
try:
group_session = requests.Session()
group_session.trust_env = False
group_auth = Auth(group_session)
# Reutiliza o token da loja para evitar nova ida ao endpoint /api/tokens por grupo.
group_auth.override_bearer = store_bearer
group_page = first_page
all_group_installments: List[Dict[str, Any]] = []
group_total = 0
group_total_pages = 1
while True:
group_body = get_installments_page(
session=group_session,
auth=group_auth,
start_date=None,
end_date=None,
installment_change=None,
mediator_code=None,
page=group_page,
installment_group_code=group_code,
cookie_header=cookie_header,
)
group_page_items = (((group_body.get("data") or {}).get("installments")) or [])
all_group_installments.extend(group_page_items)
group_pagination = ((group_body.get("data") or {}).get("pagination") or {})
group_total = int(group_pagination.get("total") or len(all_group_installments))
group_limit = int(group_pagination.get("limit") or len(group_page_items) or 1)
group_total_pages = (
max(1, (group_total + group_limit - 1) // group_limit) if group_total else group_page
)
print(
f"[grupo-pagina] loja={mediator_code} installmentGroupCode={group_code} "
f"pagina={group_page}/{group_total_pages} itens_pagina={len(group_page_items)}"
)
if not group_page_items:
break
if group_page >= group_total_pages:
break
if group_page - first_page + 1 >= max_pages_per_query:
print(
f"[stop] grupo={group_code} limite MAX_PAGES_PER_QUERY={max_pages_per_query} atingido"
)
break
group_page += 1
group_installments = _dedupe_installments(all_group_installments)
count = len(group_installments)
print(f"[grupo-ok] loja={mediator_code} installmentGroupCode={group_code} itens={count}")
group_body_agg = {
"data": {
"installments": group_installments,
"pagination": {"limit": count, "total": group_total or count},
},
"status": 200,
"message": "Success",
}
return {
"pedidoNumero": group_code,
"consulta": {
"installmentGroupCode": group_code,
"pageStart": first_page,
"pagesFetched": group_total_pages,
},
"ok": True,
"totalItens": count,
"detalhes": group_body_agg,
}
except Exception as e:
print(f"[grupo-erro] loja={mediator_code} installmentGroupCode={group_code} erro={e}")
return {
"pedidoNumero": group_code,
"consulta": {
"installmentGroupCode": group_code,
"pageStart": first_page,
},
"ok": False,
"error": str(e),
}
if group_codes:
with ThreadPoolExecutor(max_workers=max(1, group_workers)) as group_pool:
group_futures = {group_pool.submit(fetch_group, gc): gc for gc in group_codes}
for fut in as_completed(group_futures):
gc = group_futures[fut]
pedidos[gc] = fut.result()
store_out = {
"queryWindow": {
@@ -705,35 +805,19 @@ def main() -> None:
"endInstallmentChangeDate": end_date,
"installmentChange": installment_change,
"pageStart": first_page,
"pagesFetched": total_pages,
"pagesFetched": pages_fetched_total,
"windowsFetched": len(windows),
"chunkDays": chunk_days if chunk_days > 0 else None,
},
"mediatorCode": mediator_code,
"pedidosEncontradosPagina": len(group_codes),
"pedidos": pedidos,
"pedidosEncontradosPagina": len(all_group_codes_seen),
"grupos_unicos": len(all_group_codes_seen),
}
if write_sql:
sql_rows: List[Dict[str, Any]] = []
for gc, pedido_payload in pedidos.items():
if not pedido_payload.get("ok"):
continue
raw_response = pedido_payload.get("detalhes") or {}
sql_rows.append(
{
"mediatorCode": mediator_code,
"installmentGroupCode": gc,
"rawResponse": raw_response,
"installments": (((raw_response.get("data") or {}).get("installments")) or []),
}
)
stats = upsert_doc_pedidos_sqlserver(
sql_rows,
sql_conn,
mediator_code_log=mediator_code,
)
store_out["sqlUpsert"] = stats
store_out["sqlUpsert"] = {"pedidos": total_sql_pedidos, "parcelas": total_sql_parcelas}
print(
f"[sql] loja={mediator_code} pedidos_upsert={stats.get('pedidos')} "
f"parcelas_upsert={stats.get('parcelas')}"
f"[sql] loja={mediator_code} pedidos_upsert={total_sql_pedidos} "
f"parcelas_upsert={total_sql_parcelas}"
)
if save_json:
store_file = os.path.join(output_dir, f"installments_loja_{mediator_code}.json")
@@ -781,6 +865,31 @@ def main() -> None:
failed[mediator_code] = f"erro no worker: {e}"
print(f"[falha] loja={mediator_code} erro no worker: {e}")
retry_wait_sec = int(os.getenv("RETRY_FAILED_WAIT_SEC", "90"))
if failed and retry_wait_sec > 0:
retry_candidates = sorted(
k for k, v in failed.items() if "429" in str(v) or "temporaria" in str(v).lower()
)
if retry_candidates:
print(
f"[retry] {len(retry_candidates)} loja(s) falharam por throttle. "
f"Aguardando {retry_wait_sec}s antes de retentar sequencialmente..."
)
time.sleep(retry_wait_sec)
for mc in retry_candidates:
result = process_store(mc)
status = result["status"]
if status == "authorized":
authorized.append(mc)
failed.pop(mc, None)
if incremental_mode:
watermark[str(mc)] = str(result["endDate"])
save_watermark(watermark_file, watermark)
print(f"[retry-ok] loja={mc}")
else:
failed[mc] = str(result.get("error") or "erro desconhecido")
print(f"[retry-falha] loja={mc} erro={failed[mc]}")
print(
"[resumo] "
f"lojas_total={len(stores)} autorizadas={len(authorized)} "