34037 - Refresh Data Dev

jobs/colin-extract-refresh/Dockerfile

@@ -40,7 +40,7 @@
 RUN set -eux; \
     apt-get update; \
     apt-get install -y --no-install-recommends default-jre-headless curl;\
-    curl -fsSL "https://www.dbschema.com/download/dbschema_unix_9_7_1.tar.gz" -o /tmp/dbschema.tar.gz;\
+    curl --http1.1 -fsSL "https://www.dbschema.com/download/dbschema_unix_9_7_1.tar.gz" -o /tmp/dbschema.tar.gz;\
     tar -xzf /tmp/dbschema.tar.gz -C /opt; \
     rm /tmp/dbschema.tar.gz; \
     mkdir -p "${DBSCHEMA_HOME}/lib"; \
@@ -50,9 +50,21 @@
     rm -rf /var/lib/apt/lists/*
 
 ENV HOME=/opt/app-root
-RUN mkdir -p /opt/app-root/.DbSchema/cli /opt/app-root/.DbSchema/drivers/PostgreSql && \
-    curl -fsSL "https://repo1.maven.org/maven2/org/postgresql/postgresql/42.7.4/postgresql-42.7.4.jar" -o /opt/app-root/.DbSchema/drivers/PostgreSql/postgresql.jar && \
-    chmod -R g+rwX /opt/app-root/.DbSchema && \
+
+RUN mkdir -p ${HOME}/.DbSchema/cli ${HOME}/.DbSchema/drivers/PostgreSql ${HOME}/.DbSchema/drivers/Oracle && \
+    curl -fsSL "https://repo1.maven.org/maven2/org/postgresql/postgresql/42.7.4/postgresql-42.7.4.jar" -o ${HOME}/.DbSchema/drivers/PostgreSql/postgresql.jar && \
+    printf '%s\n' 'download driver oracle' > /tmp/download_oracle_driver.sql && \
+    dbschemacli /tmp/download_oracle_driver.sql && \
+    if [ -f /root/.DbSchema/drivers/oracle/driver.zip ]; then \
+        unzip -o /root/.DbSchema/drivers/oracle/driver.zip -d ${HOME}/.DbSchema/drivers/Oracle/; \
+    elif [ -f ${HOME}/.DbSchema/drivers/oracle/driver.zip ]; then \
+        unzip -o ${HOME}/.DbSchema/drivers/oracle/driver.zip -d ${HOME}/.DbSchema/drivers/Oracle/; \
+    else \
+        echo "Oracle JDBC driver download failed" >&2; exit 1; \
+    fi && \
+    rm -rf /root/.DbSchema/drivers/oracle ${HOME}/.DbSchema/drivers/oracle/driver.zip
+
+RUN chmod -R g+rwX ${HOME}/.DbSchema && \
     chmod 755 /opt/app-root
 
 WORKDIR /opt/app-root
@@ -64,7 +76,7 @@
 
 ENV PATH="${DBSCHEMA_HOME}:${PATH}"
 
-COPY src/ ./colin-exrtact-refresh/src/
+COPY src/ ./colin-extract-refresh/src/
 RUN mkdir -p /opt/app-root/colin-extract-refresh/src/subset/generated && \
     chmod -R 777 /opt/app-root/colin-extract-refresh/src/subset/generated /opt/app-root/.DbSchema
 
@@ -74,5 +86,4 @@
 
 COPY src .
 
-ENTRYPOINT ["python", "generate_cprd_subset_extract.py"]
-CMD ["--corp-file", "corp.txt", "--mode", "refresh"]
+CMD ["/bin/sh", "-c", "python /opt/app-root/colin-extract-refresh/src/test_connectivity.py && python /opt/app-root/colin-extract-refresh/src/generate_cprd_subset_extract.py --corp-file corp.txt --target-connection my_proxy_test --target-schema colin_extract_temp --mode refresh  && dbschemacli /opt/app-root/colin-extract-refresh/src/subset/generated/subset_refresh.sql"]

jobs/colin-extract-refresh/src/corp.txt

@@ -0,0 +1 @@
+BC0009721

jobs/colin-extract-refresh/src/dbschemacli_init.py

@@ -11,11 +11,18 @@ def write_dbschema_init(cfg: _Config) -> Path:
     host = cfg.DB_HOST_COLIN_MIGR
     port = int(cfg.DB_PORT_COLIN_MIGR)
     name = cfg.DB_NAME_COLIN_MIGR
+
+    ora_user = cfg.DB_USER_COLIN_ORACLE
+    ora_password = cfg.DB_PASSWORD_COLIN_ORACLE
+    ora_host = cfg.DB_HOST_COLIN_ORACLE
+    ora_port = int(cfg.DB_PORT_COLIN_ORACLE)
+    ora_name = cfg.DB_NAME_COLIN_ORACLE
 
     lines = [
         'register driver PostgreSql org.postgresql.Driver jdbc:postgresql://<host>:<port>/<db>?reWriteBatchedInserts=true "port=5432"',
+        'register driver Oracle oracle.jdbc.OracleDriver jdbc:oracle:thin:@<host>:<port>:<db> "port=1521"',
         f'connection my_proxy_test -d PostgreSql -u {user} -p {password} -h {host} -P {port} -D {name}',
-
+        f'connection cprd -d Oracle -u {ora_user} -p {ora_password} -h {ora_host} -P {ora_port} -D {ora_name}',
     ]
 
     init_path.write_text('\n'.join(lines) + '\n')

jobs/colin-extract-refresh/src/generate_cprd_subset_extract.py

@@ -592,9 +592,9 @@
         lines.append(f"execute {tmpl_resolve_execute_path(templates.disable_triggers, out_dir=cfg.out_chunks_dir).as_posix()}")
         if cfg.mode == cfg_GenerationMode.REFRESH:
             lines.append("-- Refresh-only: preserved processing/tracking tables still reference corporation/event rows.")
-            lines.append("ALTER TABLE corp_processing DISABLE TRIGGER ALL;")
-            # lines.append("ALTER TABLE auth_processing DISABLE TRIGGER ALL;")
-            lines.append("ALTER TABLE colin_tracking DISABLE TRIGGER ALL;")
+            # lines.append("ALTER TABLE corp_processing DISABLE TRIGGER ALL;")
+            # # lines.append("ALTER TABLE auth_processing DISABLE TRIGGER ALL;")
+            # lines.append("ALTER TABLE colin_tracking DISABLE TRIGGER ALL;")
         lines.append("")
         return
 
@@ -612,9 +612,9 @@
         lines.append(f"execute {tmpl_resolve_execute_path(templates.enable_triggers, out_dir=cfg.out_chunks_dir).as_posix()}")
         if cfg.mode == cfg_GenerationMode.REFRESH:
             lines.append("-- Refresh-only: restore preserved processing/tracking table triggers too.")
-            lines.append("ALTER TABLE corp_processing ENABLE TRIGGER ALL;")
-            # lines.append("ALTER TABLE auth_processing ENABLE TRIGGER ALL;")
-            lines.append("ALTER TABLE colin_tracking ENABLE TRIGGER ALL;")
+            # lines.append("ALTER TABLE corp_processing ENABLE TRIGGER ALL;")
+            # # lines.append("ALTER TABLE auth_processing ENABLE TRIGGER ALL;")
+            # lines.append("ALTER TABLE colin_tracking ENABLE TRIGGER ALL;")
         lines.append("")
         return
 

jobs/colin-extract-refresh/src/test_connectivity.py

@@ -6,7 +6,7 @@
 def main() -> int:
     print("== running test_connectivity.py ==")
     run_business_check()
-    # run_colin_check()
+    run_colin_check()
     print("== done test_connectivity.py ==")
     return 0