diff --git a/jobs/colin-extract-refresh/Dockerfile b/jobs/colin-extract-refresh/Dockerfile index 7f70d720d0..ced0de22f1 100644 --- a/jobs/colin-extract-refresh/Dockerfile +++ b/jobs/colin-extract-refresh/Dockerfile @@ -40,7 +40,7 @@ ENV DBSCHEMA_VERSION=9_7_1 RUN set -eux; \ apt-get update; \ apt-get install -y --no-install-recommends default-jre-headless curl;\ - curl -fsSL "https://www.dbschema.com/download/dbschema_unix_9_7_1.tar.gz" -o /tmp/dbschema.tar.gz;\ + curl --http1.1 -fsSL "https://www.dbschema.com/download/dbschema_unix_9_7_1.tar.gz" -o /tmp/dbschema.tar.gz;\ tar -xzf /tmp/dbschema.tar.gz -C /opt; \ rm /tmp/dbschema.tar.gz; \ mkdir -p "${DBSCHEMA_HOME}/lib"; \ @@ -50,9 +50,21 @@ RUN set -eux; \ rm -rf /var/lib/apt/lists/* ENV HOME=/opt/app-root -RUN mkdir -p /opt/app-root/.DbSchema/cli /opt/app-root/.DbSchema/drivers/PostgreSql && \ - curl -fsSL "https://repo1.maven.org/maven2/org/postgresql/postgresql/42.7.4/postgresql-42.7.4.jar" -o /opt/app-root/.DbSchema/drivers/PostgreSql/postgresql.jar && \ - chmod -R g+rwX /opt/app-root/.DbSchema && \ + +RUN mkdir -p ${HOME}/.DbSchema/cli ${HOME}/.DbSchema/drivers/PostgreSql ${HOME}/.DbSchema/drivers/Oracle && \ + curl -fsSL "https://repo1.maven.org/maven2/org/postgresql/postgresql/42.7.4/postgresql-42.7.4.jar" -o ${HOME}/.DbSchema/drivers/PostgreSql/postgresql.jar && \ + printf '%s\n' 'download driver oracle' > /tmp/download_oracle_driver.sql && \ + dbschemacli /tmp/download_oracle_driver.sql && \ + if [ -f /root/.DbSchema/drivers/oracle/driver.zip ]; then \ + unzip -o /root/.DbSchema/drivers/oracle/driver.zip -d ${HOME}/.DbSchema/drivers/Oracle/; \ + elif [ -f ${HOME}/.DbSchema/drivers/oracle/driver.zip ]; then \ + unzip -o ${HOME}/.DbSchema/drivers/oracle/driver.zip -d ${HOME}/.DbSchema/drivers/Oracle/; \ + else \ + echo "Oracle JDBC driver download failed" >&2; exit 1; \ + fi && \ + rm -rf /root/.DbSchema/drivers/oracle ${HOME}/.DbSchema/drivers/oracle/driver.zip + +RUN chmod -R g+rwX ${HOME}/.DbSchema && \ chmod 755 /opt/app-root WORKDIR /opt/app-root @@ -64,7 +76,7 @@ RUN apt-get update; \ ENV PATH="${DBSCHEMA_HOME}:${PATH}" -COPY src/ ./colin-exrtact-refresh/src/ +COPY src/ ./colin-extract-refresh/src/ RUN mkdir -p /opt/app-root/colin-extract-refresh/src/subset/generated && \ chmod -R 777 /opt/app-root/colin-extract-refresh/src/subset/generated /opt/app-root/.DbSchema @@ -74,5 +86,4 @@ EXPOSE 8080 COPY src . -ENTRYPOINT ["python", "generate_cprd_subset_extract.py"] -CMD ["--corp-file", "corp.txt", "--mode", "refresh"] +CMD ["/bin/sh", "-c", "python /opt/app-root/colin-extract-refresh/src/test_connectivity.py && python /opt/app-root/colin-extract-refresh/src/generate_cprd_subset_extract.py --corp-file corp.txt --target-connection my_proxy_test --target-schema colin_extract_temp --mode refresh && dbschemacli /opt/app-root/colin-extract-refresh/src/subset/generated/subset_refresh.sql"] \ No newline at end of file diff --git a/jobs/colin-extract-refresh/src/corp.txt b/jobs/colin-extract-refresh/src/corp.txt new file mode 100644 index 0000000000..49098caf87 --- /dev/null +++ b/jobs/colin-extract-refresh/src/corp.txt @@ -0,0 +1 @@ +BC0009721 \ No newline at end of file diff --git a/jobs/colin-extract-refresh/src/dbschemacli_init.py b/jobs/colin-extract-refresh/src/dbschemacli_init.py index 33bf617f57..ce31ac1fd3 100644 --- a/jobs/colin-extract-refresh/src/dbschemacli_init.py +++ b/jobs/colin-extract-refresh/src/dbschemacli_init.py @@ -11,11 +11,18 @@ def write_dbschema_init(cfg: _Config) -> Path: host = cfg.DB_HOST_COLIN_MIGR port = int(cfg.DB_PORT_COLIN_MIGR) name = cfg.DB_NAME_COLIN_MIGR + + ora_user = cfg.DB_USER_COLIN_ORACLE + ora_password = cfg.DB_PASSWORD_COLIN_ORACLE + ora_host = cfg.DB_HOST_COLIN_ORACLE + ora_port = int(cfg.DB_PORT_COLIN_ORACLE) + ora_name = cfg.DB_NAME_COLIN_ORACLE lines = [ 'register driver PostgreSql org.postgresql.Driver jdbc:postgresql://:/?reWriteBatchedInserts=true "port=5432"', + 'register driver Oracle oracle.jdbc.OracleDriver jdbc:oracle:thin:@:: "port=1521"', f'connection my_proxy_test -d PostgreSql -u {user} -p {password} -h {host} -P {port} -D {name}', - + f'connection cprd -d Oracle -u {ora_user} -p {ora_password} -h {ora_host} -P {ora_port} -D {ora_name}', ] init_path.write_text('\n'.join(lines) + '\n') diff --git a/jobs/colin-extract-refresh/src/generate_cprd_subset_extract.py b/jobs/colin-extract-refresh/src/generate_cprd_subset_extract.py index 2286e6cc9e..f33587a478 100644 --- a/jobs/colin-extract-refresh/src/generate_cprd_subset_extract.py +++ b/jobs/colin-extract-refresh/src/generate_cprd_subset_extract.py @@ -592,9 +592,9 @@ def _gen_emit_pg_disable_begin(lines: List[str], *, cfg: cfg_GenerationConfig, t lines.append(f"execute {tmpl_resolve_execute_path(templates.disable_triggers, out_dir=cfg.out_chunks_dir).as_posix()}") if cfg.mode == cfg_GenerationMode.REFRESH: lines.append("-- Refresh-only: preserved processing/tracking tables still reference corporation/event rows.") - lines.append("ALTER TABLE corp_processing DISABLE TRIGGER ALL;") - # lines.append("ALTER TABLE auth_processing DISABLE TRIGGER ALL;") - lines.append("ALTER TABLE colin_tracking DISABLE TRIGGER ALL;") + # lines.append("ALTER TABLE corp_processing DISABLE TRIGGER ALL;") + # # lines.append("ALTER TABLE auth_processing DISABLE TRIGGER ALL;") + # lines.append("ALTER TABLE colin_tracking DISABLE TRIGGER ALL;") lines.append("") return @@ -612,9 +612,9 @@ def _gen_emit_pg_disable_end(lines: List[str], *, cfg: cfg_GenerationConfig, tem lines.append(f"execute {tmpl_resolve_execute_path(templates.enable_triggers, out_dir=cfg.out_chunks_dir).as_posix()}") if cfg.mode == cfg_GenerationMode.REFRESH: lines.append("-- Refresh-only: restore preserved processing/tracking table triggers too.") - lines.append("ALTER TABLE corp_processing ENABLE TRIGGER ALL;") - # lines.append("ALTER TABLE auth_processing ENABLE TRIGGER ALL;") - lines.append("ALTER TABLE colin_tracking ENABLE TRIGGER ALL;") + # lines.append("ALTER TABLE corp_processing ENABLE TRIGGER ALL;") + # # lines.append("ALTER TABLE auth_processing ENABLE TRIGGER ALL;") + # lines.append("ALTER TABLE colin_tracking ENABLE TRIGGER ALL;") lines.append("") return diff --git a/jobs/colin-extract-refresh/src/test_connectivity.py b/jobs/colin-extract-refresh/src/test_connectivity.py index 452e9d9577..cdfc59630f 100644 --- a/jobs/colin-extract-refresh/src/test_connectivity.py +++ b/jobs/colin-extract-refresh/src/test_connectivity.py @@ -6,7 +6,7 @@ def main() -> int: print("== running test_connectivity.py ==") run_business_check() - # run_colin_check() + run_colin_check() print("== done test_connectivity.py ==") return 0