diff --git a/install/ci-vm/ci-linux/ci/runCI b/install/ci-vm/ci-linux/ci/runCI index 092d425e..f9bec89e 100644 --- a/install/ci-vm/ci-linux/ci/runCI +++ b/install/ci-vm/ci-linux/ci/runCI @@ -7,6 +7,11 @@ DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +# Enable coredump capture +ulimit -c unlimited +mkdir -p /tmp/coredumps +echo "/tmp/coredumps/core.%e.%p" | sudo tee /proc/sys/kernel/core_pattern > /dev/null + if [ ! -f "$DIR/variables" ]; then # No variable file defined sudo shutdown -h now @@ -124,8 +129,62 @@ if [ -e "${dstDir}/ccextractor" ]; then ./ccextractor --version >> "${logFile}" 2>&1 echo "=== End Version Info ===" >> "${logFile}" postStatus "testing" "Running tests" + ccextractor_path="$(pwd)/ccextractor" + combined_stdout="/tmp/combined_stdout.log" + : > "${combined_stdout}" + + # Create a wrapper script that tees stdout/stderr to a combined log + wrapper_path="$(pwd)/ccextractor_wrapper" + cat > "${wrapper_path}" << 'WRAPPER_EOF' +#!/bin/bash +COMBINED_LOG="/tmp/combined_stdout.log" +REAL_BINARY="PLACEHOLDER_BINARY" +EXIT_CODE_FILE=$(mktemp) +echo "=== TEST INVOCATION: $@ ===" >> "$COMBINED_LOG" +{ "$REAL_BINARY" "$@" 2>&1; echo $? > "$EXIT_CODE_FILE"; } | tee -a "$COMBINED_LOG" +exit_code=$(cat "$EXIT_CODE_FILE") +rm -f "$EXIT_CODE_FILE" +echo "=== EXIT CODE: ${exit_code} ===" >> "$COMBINED_LOG" +echo "" >> "$COMBINED_LOG" +exit $exit_code +WRAPPER_EOF + sed -i "s|PLACEHOLDER_BINARY|${ccextractor_path}|" "${wrapper_path}" + chmod +x "${wrapper_path}" + executeCommand cd ${suiteDstDir} - executeCommand ${tester} --debug --entries "${testFile}" --executable "ccextractor" --tempfolder "${tempFolder}" --timeout 600 --reportfolder "${reportFolder}" --resultfolder "${resultFolder}" --samplefolder "${sampleFolder}" --method Server --url "${reportURL}" + executeCommand ${tester} --debug --entries "${testFile}" --executable "${wrapper_path}" --tempfolder "${tempFolder}" --timeout 600 --reportfolder "${reportFolder}" --resultfolder "${resultFolder}" --samplefolder "${sampleFolder}" --method Server --url "${reportURL}" + + # Upload artifacts through the Sample Platform server + upload_artifact() { + local file_path="$1" + local artifact_name="$2" + if [ -f "$file_path" ]; then + local http_code + http_code=$(curl -s -A "${userAgent}" \ + --form "type=artifact" \ + --form "name=${artifact_name}" \ + --form "file=@${file_path}" \ + -w "%{http_code}" -o /dev/null \ + "${reportURL}" 2>/dev/null) + if [ -z "$http_code" ] || [ "$http_code" -lt 200 ] || [ "$http_code" -ge 300 ]; then + echo "Artifact upload failed for ${artifact_name}: HTTP ${http_code:-no_response}" >> "${logFile}" + fi + fi + } + + upload_artifact "$ccextractor_path" "ccextractor" + + # Upload combined stdout log + upload_artifact "${combined_stdout}" "combined_stdout.log" + + # Upload coredumps if any + for core_file in /tmp/coredumps/core.*; do + if [ -f "$core_file" ]; then + upload_artifact "$core_file" "coredump" + break + fi + done + sendLogFile postStatus "completed" "Ran all tests" diff --git a/mod_ci/controllers.py b/mod_ci/controllers.py index 618d3151..ae2aceff 100755 --- a/mod_ci/controllers.py +++ b/mod_ci/controllers.py @@ -1194,8 +1194,7 @@ def create_instance(compute, project, zone, test, reportURL) -> Dict: startup_script = f.read() metadata_items = [ {'key': 'startup-script', 'value': startup_script}, - {'key': 'reportURL', 'value': reportURL}, - {'key': 'bucket', 'value': config.get('GCS_BUCKET_NAME', '')} + {'key': 'reportURL', 'value': reportURL} ] elif test.platform == TestPlatform.windows: image_response = compute.images().getFromFamily(project=config.get('WINDOWS_INSTANCE_PROJECT_NAME', ''), @@ -1216,8 +1215,7 @@ def create_instance(compute, project, zone, test, reportURL) -> Dict: {'key': 'windows-startup-script-ps1', 'value': startup_script}, {'key': 'service_account', 'value': service_account}, {'key': 'rclone_conf', 'value': rclone_conf}, - {'key': 'reportURL', 'value': reportURL}, - {'key': 'bucket', 'value': config.get('GCS_BUCKET_NAME', '')} + {'key': 'reportURL', 'value': reportURL} ] source_disk_image = image_response['selfLink'] @@ -2344,6 +2342,11 @@ def progress_reporter(test_id, token): if not upload_type_request(log, test_id, repo_folder, test, request): return "EMPTY" + elif request.form['type'] == 'artifact': + log.info(f'[PROGRESS_REPORTER][Test: {test_id}] Artifact upload') + if not artifact_upload_request(log, test_id, request): + return "EMPTY" + elif request.form['type'] == 'finish': log.info(f'[PROGRESS_REPORTER][Test: {test_id}] Test finished') finish_type_request(log, test_id, test, request) @@ -2635,7 +2638,7 @@ def upload_log_type_request(log, test_id, repo_folder, test, request) -> bool: uploaded_file.save(temp_path) final_path = os.path.join(repo_folder, 'LogFiles', f"{test.id}.txt") - os.rename(temp_path, final_path) + os.replace(temp_path, final_path) log.debug("Stored log file") return True @@ -2681,7 +2684,7 @@ def upload_type_request(log, test_id, repo_folder, test, request) -> bool: results_dir = os.path.join(repo_folder, 'TestResults') os.makedirs(results_dir, exist_ok=True) final_path = os.path.join(results_dir, f'{file_hash}{file_extension}') - os.rename(temp_path, final_path) + os.replace(temp_path, final_path) rto = RegressionTestOutput.query.filter( RegressionTestOutput.id == request.form['test_file_id']).first() result_file = TestResultFile(test.id, request.form['test_id'], rto.id, rto.correct, file_hash) @@ -2693,6 +2696,46 @@ def upload_type_request(log, test_id, repo_folder, test, request) -> bool: return False +# Allowed artifact names that the VM can upload +from mod_test.controllers import CCEXTRACTOR_LINUX_BINARY, CCEXTRACTOR_WIN_BINARY +ALLOWED_ARTIFACT_NAMES = {CCEXTRACTOR_LINUX_BINARY, CCEXTRACTOR_WIN_BINARY, 'combined_stdout.log', 'coredump'} + + +def artifact_upload_request(log, test_id, request) -> bool: + """ + Handle artifact upload from the CI VM. + + Validates the artifact name against an allow-list, then uploads + the file to GCS under test_artifacts/{test_id}/{name}. + + :param log: logger + :type log: Logger + :param test_id: The id of the test to update. + :type test_id: int + :param request: Request parameters + :type request: Request + :return: True if upload succeeded, False otherwise. + :rtype: bool + """ + from run import storage_client_bucket + + artifact_name = request.form.get('name', '') + if artifact_name not in ALLOWED_ARTIFACT_NAMES: + log.warning(f"[Test: {test_id}] Rejected artifact upload with disallowed name: {artifact_name}") + return False + + if 'file' not in request.files: + log.warning(f"[Test: {test_id}] Artifact upload missing file") + return False + + uploaded_file = request.files['file'] + blob_path = f'test_artifacts/{test_id}/{artifact_name}' + blob = storage_client_bucket.blob(blob_path) + blob.upload_from_file(uploaded_file.stream) + log.info(f"[Test: {test_id}] Artifact '{artifact_name}' uploaded to {blob_path}") + return True + + def finish_type_request(log, test_id, test, request): """ Handle finish request type for progress reporter. diff --git a/mod_test/controllers.py b/mod_test/controllers.py index e80c0a47..26aabc23 100644 --- a/mod_test/controllers.py +++ b/mod_test/controllers.py @@ -21,6 +21,9 @@ mod_test = Blueprint('test', __name__) +CCEXTRACTOR_WIN_BINARY = 'ccextractorwinfull.exe' +CCEXTRACTOR_LINUX_BINARY = 'ccextractor' + @mod_test.before_app_request def before_app_request() -> None: @@ -379,15 +382,17 @@ def download_build_log_file(test_id): :return: build log text file :rtype: Flask response """ - from run import config + from run import config, storage_client_bucket test = Test.query.filter(Test.id == test_id).first() + from flask import send_from_directory + if test is not None: file_name = f"{test_id}.txt" - log_file_path = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'LogFiles', file_name) - + log_dir = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'LogFiles') + log_file_path = os.path.join(log_dir, file_name) if os.path.isfile(log_file_path): - return serve_file_download(file_name, 'LogFiles') + return send_from_directory(log_dir, file_name, as_attachment=True) raise TestNotFoundException(f"Build log for Test {test_id} not found") @@ -442,3 +447,249 @@ def stop_test(test_id): g.db.commit() g.log.info(f"test with id: {test_id} stopped") return redirect(url_for('.by_id', test_id=test.id)) + + +def _artifact_redirect(blob_path, filename='artifact'): + """Generate a signed URL for a GCS artifact and redirect, or 404.""" + from datetime import timedelta + + from run import config, storage_client_bucket + + blob = storage_client_bucket.blob(blob_path) + if not blob.exists(): + abort(404) + url = blob.generate_signed_url( + version="v4", + expiration=timedelta(minutes=config.get('GCS_SIGNED_URL_EXPIRY_LIMIT', 30)), + method="GET", + response_disposition=f'attachment; filename="{filename}"' + ) + return redirect(url) + + +@mod_test.route('//binary', methods=['GET']) +@login_required +def download_binary(test_id): + """Download the ccextractor binary used in a test.""" + test = Test.query.filter(Test.id == test_id).first() + if test is None: + abort(404) + name = CCEXTRACTOR_LINUX_BINARY if test.platform == TestPlatform.linux else CCEXTRACTOR_WIN_BINARY + return _artifact_redirect(f'test_artifacts/{test_id}/{name}', filename=name) + + +@mod_test.route('//coredump', methods=['GET']) +@login_required +def download_coredump(test_id): + """Download the coredump from a test, if one was produced.""" + return _artifact_redirect( + f'test_artifacts/{test_id}/coredump', + filename=f'coredump-{test_id}' + ) + + +@mod_test.route('//combined-stdout', methods=['GET']) +@login_required +def download_combined_stdout(test_id): + """Download the combined stdout/stderr log from all test invocations.""" + return _artifact_redirect( + f'test_artifacts/{test_id}/combined_stdout.log', + filename=f'combined_stdout-{test_id}.log' + ) + + +@mod_test.route('//regression///output-got', methods=['GET']) +@login_required +def download_output_got(test_id, regression_test_id, output_id): + """Download the actual output file from TestResults using DB hash.""" + rf = TestResultFile.query.filter(and_( + TestResultFile.test_id == test_id, + TestResultFile.regression_test_id == regression_test_id, + TestResultFile.regression_test_output_id == output_id + )).first() + if rf is None or rf.got is None: + abort(404) + ext = os.path.splitext(rf.regression_test_output.filename_correct)[1] + return _artifact_redirect( + f'TestResults/{rf.got}{ext}', + filename=f'output_got_{regression_test_id}_{output_id}{ext}' + ) + + +@mod_test.route('//regression///output-expected', methods=['GET']) +@login_required +def download_output_expected(test_id, regression_test_id, output_id): + """Download the expected output file from TestResults using DB hash.""" + rf = TestResultFile.query.filter(and_( + TestResultFile.test_id == test_id, + TestResultFile.regression_test_id == regression_test_id, + TestResultFile.regression_test_output_id == output_id + )).first() + if rf is None: + abort(404) + ext = os.path.splitext(rf.regression_test_output.filename_correct)[1] + return _artifact_redirect( + f'TestResults/{rf.expected}{ext}', + filename=f'output_expected_{regression_test_id}_{output_id}{ext}' + ) +@mod_test.route('//sample/', methods=['GET']) +@login_required +def download_sample_ai(test_id, sample_id): + """Download the sample file for a regression test.""" + from mod_sample.models import Sample + sample = Sample.query.filter(Sample.id == sample_id).first() + if sample is None: + abort(404) + return _artifact_redirect( + f'TestFiles/{sample.filename}', + filename=sample.original_name + ) + + +def _build_output_entry(test_id, rt, expected_output, result_files): + """Build a single output entry dict for the ai.json response.""" + matched_rf = next( + (rf for rf in result_files + if rf.test_id != -1 and rf.regression_test_output_id == expected_output.id), + None + ) + + got_url = None + diff_url = None + + if matched_rf and matched_rf.got is not None: + got_url = url_for( + '.download_output_got', + test_id=test_id, + regression_test_id=rt.id, + output_id=expected_output.id, + _external=True + ) + diff_url = url_for( + '.generate_diff', + test_id=test_id, + regression_test_id=rt.id, + output_id=expected_output.id, + to_view=0, + _external=True + ) + + return { + 'output_id': expected_output.id, + 'correct_extension': expected_output.correct_extension, + 'expected_url': url_for( + '.download_output_expected', + test_id=test_id, + regression_test_id=rt.id, + output_id=expected_output.id, + _external=True + ), + 'got_url': got_url, + 'diff_url': diff_url, + } + + +def _process_test_case(test, category_name, t_data): + """Build a structured dict for a single test case in the ai.json response.""" + rt = t_data['test'] + result = t_data['result'] + is_error = t_data.get('error', False) + result_files = t_data['files'] + + outputs = [ + _build_output_entry(test.id, rt, expected_output, result_files) + for expected_output in rt.output_files + if not expected_output.ignore + ] + + test_case = { + 'regression_test_id': rt.id, + 'category': category_name, + 'sample_filename': rt.sample.original_name, + 'sample_url': url_for( + '.download_sample_ai', + test_id=test.id, + sample_id=rt.sample.id, + _external=True + ), + 'arguments': rt.command, + 'result': 'Fail' if is_error else 'Pass', + 'exit_code': result.exit_code if result else None, + 'expected_exit_code': result.expected_rc if result else None, + 'runtime_ms': result.runtime if result else None, + 'outputs': outputs, + } + + # Format the reproduction command based on platform + binary_name = f'./{CCEXTRACTOR_LINUX_BINARY}' if test.platform == TestPlatform.linux else CCEXTRACTOR_WIN_BINARY + test_case['how_to_reproduce'] = f'{binary_name} {rt.command} {rt.sample.original_name}' + + return test_case + + +@mod_test.route('//ai.json', methods=['GET']) +@login_required +def ai_json_endpoint(test_id): + """Structured JSON with download URLs for all artifacts — for AI agents.""" + from run import storage_client_bucket + + test = Test.query.filter(Test.id == test_id).first() + if test is None: + return jsonify({'error': f'Test {test_id} not found'}), 404 + + def blob_exists(path): + return storage_client_bucket.blob(path).exists() + + binary_name = CCEXTRACTOR_LINUX_BINARY if test.platform == TestPlatform.linux else CCEXTRACTOR_WIN_BINARY + has_binary = blob_exists(f'test_artifacts/{test_id}/{binary_name}') + has_coredump = blob_exists(f'test_artifacts/{test_id}/coredump') + has_combined_stdout = blob_exists(f'test_artifacts/{test_id}/combined_stdout.log') + + results = get_test_results(test) + test_cases = [] + total = 0 + passed = 0 + failed = 0 + + for category in results: + for t_data in category['tests']: + total += 1 + if t_data.get('error', False): + failed += 1 + else: + passed += 1 + + test_cases.append(_process_test_case(test, category['category'].name, t_data)) + + report = { + 'test_id': test.id, + 'commit': test.commit, + 'platform': test.platform.value, + 'branch': test.branch, + 'status': 'completed' if test.finished else 'running', + 'binary_url': url_for( + '.download_binary', test_id=test_id, _external=True + ) if has_binary else None, + 'coredump_url': url_for( + '.download_coredump', test_id=test_id, _external=True + ) if has_coredump else None, + 'log_url': url_for( + '.download_build_log_file', test_id=test_id, _external=True + ), + 'combined_stdout_url': url_for( + '.download_combined_stdout', test_id=test_id, _external=True + ) if has_combined_stdout else None, + 'summary': { + 'total': total, + 'passed': passed, + 'failed': failed, + }, + 'test_cases': test_cases, + 'how_to_reproduce': ( + 'Download the binary and sample, then run: ' + + (f'./{CCEXTRACTOR_LINUX_BINARY} {{arguments}} {{sample_filename}}' if test.platform.value == 'linux' + else f'{CCEXTRACTOR_WIN_BINARY} {{arguments}} {{sample_filename}}') + ), + } + + return jsonify(report) diff --git a/utility.py b/utility.py index 98eeec53..25b85d1a 100644 --- a/utility.py +++ b/utility.py @@ -30,14 +30,13 @@ def serve_file_download(file_name, file_folder, file_sub_folder='') -> werkzeug. """ from run import config, storage_client_bucket - file_path = path.join(file_folder, file_sub_folder, file_name) + file_path = '/'.join(filter(None, [file_folder, file_sub_folder, file_name])) blob = storage_client_bucket.blob(file_path) - blob.content_disposition = f'attachment; filename="{file_name}"' - blob.patch() url = blob.generate_signed_url( version="v4", - expiration=timedelta(minutes=config.get('GCS_SIGNED_URL_EXPIRY_LIMIT', '')), + expiration=timedelta(minutes=config.get('GCS_SIGNED_URL_EXPIRY_LIMIT', 30)), method="GET", + response_disposition=f'attachment; filename="{file_name}"' ) return redirect(url)