From 18f7dfe27b62a5e3a8d16ccaff5095687c54e573 Mon Sep 17 00:00:00 2001 From: dankmeme01 <42031238+dankmeme01@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:38:04 +0200 Subject: [PATCH 1/4] add loader versions to cdn --- ...dd_download_link_to_geode_version.down.sql | 1 + ..._add_download_link_to_geode_version.up.sql | 1 + src/config.rs | 10 +- src/database/repository/geode_versions.rs | 22 +++ src/database/repository/mod.rs | 1 + src/endpoints/loader.rs | 9 +- src/endpoints/mod_versions.rs | 2 +- src/main.rs | 4 +- src/s3_worker.rs | 127 +++++++++++++++++- src/types/models/loader_version.rs | 34 ++++- 10 files changed, 190 insertions(+), 21 deletions(-) create mode 100644 migrations/20260721182316_add_download_link_to_geode_version.down.sql create mode 100644 migrations/20260721182316_add_download_link_to_geode_version.up.sql create mode 100644 src/database/repository/geode_versions.rs diff --git a/migrations/20260721182316_add_download_link_to_geode_version.down.sql b/migrations/20260721182316_add_download_link_to_geode_version.down.sql new file mode 100644 index 00000000..52d0dc5a --- /dev/null +++ b/migrations/20260721182316_add_download_link_to_geode_version.down.sql @@ -0,0 +1 @@ +ALTER TABLE geode_versions DROP COLUMN downloads; diff --git a/migrations/20260721182316_add_download_link_to_geode_version.up.sql b/migrations/20260721182316_add_download_link_to_geode_version.up.sql new file mode 100644 index 00000000..cf96befd --- /dev/null +++ b/migrations/20260721182316_add_download_link_to_geode_version.up.sql @@ -0,0 +1 @@ +ALTER TABLE geode_versions ADD COLUMN downloads JSONB; diff --git a/src/config.rs b/src/config.rs index 2fe89384..bfa948fe 100644 --- a/src/config.rs +++ b/src/config.rs @@ -28,7 +28,7 @@ pub struct AppData { static_storage: PublicDisk, public_storage: PublicDisk, private_storage: PrivateDisk, - mod_storage: Option, + cdn_storage: Option, disable_downloads: bool, max_download_mb: u32, port: u16, @@ -79,7 +79,7 @@ pub async fn build_config() -> anyhow::Result { .time_to_live(Duration::from_mins(10)) .build(); - let mod_storage = if let Some(s3_config) = S3Configuration::from_env()? { + let cdn_storage = if let Some(s3_config) = S3Configuration::from_env()? { let backend = Arc::new(S3Backend::new(&s3_config)?); Some(PublicDisk::new(backend, s3_config.public_url)) } else { @@ -113,7 +113,7 @@ pub async fn build_config() -> anyhow::Result { format!("{app_url}/storage"), ), private_storage: PrivateDisk::new(Arc::new(LocalBackend::new("storage/private"))), - mod_storage, + cdn_storage, disable_downloads, max_download_mb, port, @@ -192,8 +192,8 @@ impl AppData { &self.private_storage } - pub fn mod_storage(&self) -> Option<&PublicDisk> { - self.mod_storage.as_ref() + pub fn cdn_storage(&self) -> Option<&PublicDisk> { + self.cdn_storage.as_ref() } pub fn mods_cache(&self) -> &Cache>> { diff --git a/src/database/repository/geode_versions.rs b/src/database/repository/geode_versions.rs new file mode 100644 index 00000000..014853b8 --- /dev/null +++ b/src/database/repository/geode_versions.rs @@ -0,0 +1,22 @@ +use sqlx::PgConnection; + +use crate::{database::DatabaseError, types::models::loader_version::LoaderDownloads}; + +#[tracing::instrument(skip_all, fields(tag = %tag, downloads = ?downloads))] +pub async fn update_downloads( + tag: &str, + downloads: &LoaderDownloads, + conn: &mut PgConnection, +) -> Result<(), DatabaseError> { + sqlx::query!( + "UPDATE geode_versions SET downloads = $1 WHERE tag = $2", + // should never fail + &serde_json::to_value(downloads).expect("serialization failed"), + tag + ) + .execute(&mut *conn) + .await + .inspect_err(|e| tracing::error!("{:?}", e))?; + + Ok(()) +} diff --git a/src/database/repository/mod.rs b/src/database/repository/mod.rs index 72539227..e15fe17c 100644 --- a/src/database/repository/mod.rs +++ b/src/database/repository/mod.rs @@ -2,6 +2,7 @@ pub mod auth_tokens; pub mod dependencies; pub mod deprecations; pub mod developers; +pub mod geode_versions; pub mod github_login_attempts; pub mod github_web_logins; pub mod incompatibilities; diff --git a/src/endpoints/loader.rs b/src/endpoints/loader.rs index 34345668..88e9b207 100644 --- a/src/endpoints/loader.rs +++ b/src/endpoints/loader.rs @@ -1,11 +1,12 @@ use actix_web::{HttpResponse, Responder, get, post, web}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use std::str::FromStr; use utoipa::{IntoParams, ToSchema}; use sqlx::Acquire; use crate::endpoints::ApiError; +use crate::s3_worker::S3WorkerTask; use crate::{ config::AppData, extractors::auth::Auth, @@ -122,10 +123,12 @@ pub async fn create_version( return Err(ApiError::Authorization); } + let tag = payload.tag.trim_start_matches('v').to_string(); + let mut tx = pool.begin().await?; LoaderVersion::create_version( LoaderVersionCreate { - tag: payload.tag.trim_start_matches('v').to_string(), + tag: tag.clone(), prerelease: payload.prerelease, commit_hash: payload.commit_hash.clone(), win: payload.gd.win, @@ -139,6 +142,8 @@ pub async fn create_version( tx.commit().await?; + data.send_s3_task(S3WorkerTask::UploadLoader { tag }); + Ok(HttpResponse::NoContent()) } diff --git a/src/endpoints/mod_versions.rs b/src/endpoints/mod_versions.rs index 3d48b99e..8955888c 100644 --- a/src/endpoints/mod_versions.rs +++ b/src/endpoints/mod_versions.rs @@ -241,7 +241,7 @@ pub async fn download_version( let url = mod_version .managed_download_link .as_deref() - .take_if(|_| data.mod_storage().is_some()) + .take_if(|_| data.cdn_storage().is_some()) .unwrap_or(&mod_version.download_link); if data.disable_downloads() || mod_version.status != ModVersionStatusEnum::Accepted { diff --git a/src/main.rs b/src/main.rs index 96d8d64e..8db3b405 100644 --- a/src/main.rs +++ b/src/main.rs @@ -39,8 +39,8 @@ async fn main() -> anyhow::Result<()> { let app_data = config::build_config().await?; app_data.static_storage().init().await?; app_data.private_storage().init().await?; - if let Some(mod_storage) = app_data.mod_storage() { - mod_storage.init().await?; + if let Some(cdn_storage) = app_data.cdn_storage() { + cdn_storage.init().await?; } if cli::maybe_cli(&app_data).await? { diff --git a/src/s3_worker.rs b/src/s3_worker.rs index 7a349631..cafe15c0 100644 --- a/src/s3_worker.rs +++ b/src/s3_worker.rs @@ -4,8 +4,12 @@ use actix_web::web; use bytes::Bytes; use crate::{ - config::AppData, database::repository::mod_versions::update_managed_download_link, mod_zip, - types::models::mod_gd_version::GDVersionEnum, + config::AppData, + database::repository::{ + geode_versions::update_downloads, mod_versions::update_managed_download_link, + }, + mod_zip, + types::models::{loader_version::LoaderDownloads, mod_gd_version::GDVersionEnum}, }; pub enum S3WorkerTask { @@ -15,14 +19,36 @@ pub enum S3WorkerTask { version: String, version_id: i32, }, + + UploadLoader { + tag: String, + }, } fn path_for_mod(mod_id: &str, version: &str) -> String { format!("mods/{mod_id}/{version}/{mod_id}.geode") } +fn path_for_loader(tag: &str, platform: &str) -> String { + format!("geode/{tag}/geode-v{tag}-{platform}.zip") +} + +fn path_for_resources(tag: &str) -> String { + format!("geode/{tag}/resources.zip") +} + +fn github_url_for_loader(tag: &str, platform: &str) -> String { + format!( + "https://github.com/geode-sdk/geode/releases/download/v{tag}/geode-v{tag}-{platform}.zip" + ) +} + +fn github_url_for_resources(tag: &str) -> String { + format!("https://github.com/geode-sdk/geode/releases/download/v{tag}/resources.zip") +} + async fn process_task(data: &AppData, task: S3WorkerTask) -> anyhow::Result<()> { - let storage = data.mod_storage().expect("mod storage must be set by now"); + let storage = data.cdn_storage().expect("mod storage must be set by now"); match task { S3WorkerTask::UploadMod { @@ -47,6 +73,65 @@ async fn process_task(data: &AppData, task: S3WorkerTask) -> anyhow::Result<()> public_url ); } + + S3WorkerTask::UploadLoader { tag } => { + tracing::info!("Preparing to upload Geode v{tag} to S3"); + + let mut zips = Vec::new(); + let mut downloads = LoaderDownloads::default(); + + zips.push(( + &mut downloads.win, + path_for_loader(&tag, "win"), + github_url_for_loader(&tag, "win"), + )); + zips.push(( + &mut downloads.mac, + path_for_loader(&tag, "mac"), + github_url_for_loader(&tag, "mac"), + )); + zips.push(( + &mut downloads.android32, + path_for_loader(&tag, "android32"), + github_url_for_loader(&tag, "android32"), + )); + zips.push(( + &mut downloads.android64, + path_for_loader(&tag, "android64"), + github_url_for_loader(&tag, "android64"), + )); + zips.push(( + &mut downloads.ios, + path_for_loader(&tag, "ios"), + github_url_for_loader(&tag, "ios"), + )); + zips.push(( + &mut downloads.resources, + path_for_resources(&tag), + github_url_for_resources(&tag), + )); + + for (dl, path, url) in zips { + let resp = data.http_client().get(&url).send().await?; + if let Err(e) = resp.error_for_status_ref() { + tracing::error!("Failed to download loader release from {url}: {e:?}",); + continue; + } + + let resp = resp.bytes().await?; + let public_url = storage.asset_url(&path); + storage.store(&path, &resp).await?; + + dl.url = public_url; + dl.hash = sha256::digest(&resp[..]); + } + + let mut tx = data.db().begin().await?; + update_downloads(&tag, &downloads, &mut tx).await?; + tx.commit().await?; + + tracing::info!("Uploaded new loader release to S3: {downloads:?}"); + } } Ok(()) @@ -54,7 +139,7 @@ async fn process_task(data: &AppData, task: S3WorkerTask) -> anyhow::Result<()> async fn cleanup_old_s3_files(data: &AppData) -> anyhow::Result<()> { let supported_gd = GDVersionEnum::supported_for_storage(); - let storage = data.mod_storage().expect("mod storage must be set by now"); + let storage = data.cdn_storage().expect("mod storage must be set by now"); let mut db = data.db().acquire().await?; @@ -177,8 +262,33 @@ async fn migrate_existing_mods_to_s3(data: &AppData) -> anyhow::Result<()> { Ok(()) } +async fn migrate_loader_versions_to_s3(data: &AppData) -> anyhow::Result<()> { + let mut db = data.db().acquire().await?; + + let versions = sqlx::query!("SELECT tag FROM geode_versions WHERE downloads IS NULL") + .fetch_all(&mut *db) + .await?; + + tracing::info!("Migrating {} Geode releases to S3", versions.len()); + + for record in versions { + if let Err(e) = process_task( + data, + S3WorkerTask::UploadLoader { + tag: record.tag.clone(), + }, + ) + .await + { + tracing::error!("error migrating Geode release {} to S3: {e:?}", record.tag); + } + } + + Ok(()) +} + pub async fn run_s3_worker(data: web::Data) { - if data.mod_storage().is_none() { + if data.cdn_storage().is_none() { return; } @@ -192,6 +302,13 @@ pub async fn run_s3_worker(data: web::Data) { } }); + let s_data2 = data.clone(); + tokio::spawn(async move { + if let Err(e) = migrate_loader_versions_to_s3(&s_data2).await { + tracing::error!("Error migrating loader versions to S3: {:?}", e); + } + }); + let mut interval = tokio::time::interval(Duration::from_mins(30)); loop { diff --git a/src/types/models/loader_version.rs b/src/types/models/loader_version.rs index 66883ef8..aaf4da63 100644 --- a/src/types/models/loader_version.rs +++ b/src/types/models/loader_version.rs @@ -6,12 +6,15 @@ use crate::{ }, }; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use utoipa::ToSchema; use sqlx::{ PgConnection, Postgres, QueryBuilder, - types::chrono::{DateTime, Utc}, + types::{ + Json, + chrono::{DateTime, Utc}, + }, }; #[derive(Debug)] @@ -25,6 +28,22 @@ pub struct LoaderVersionCreate { pub ios: Option, } +#[derive(Serialize, Deserialize, Default, Debug, ToSchema)] +pub struct LoaderDownload { + pub url: String, + pub hash: String, +} + +#[derive(Serialize, Deserialize, Default, Debug, ToSchema)] +pub struct LoaderDownloads { + pub win: LoaderDownload, + pub mac: LoaderDownload, + pub android32: LoaderDownload, + pub android64: LoaderDownload, + pub ios: LoaderDownload, + pub resources: LoaderDownload, +} + #[derive(Serialize, Debug, ToSchema)] pub struct LoaderVersion { pub version: String, @@ -34,6 +53,7 @@ pub struct LoaderVersion { pub commit_hash: String, #[serde(with = "chrono_dt_secs")] pub created_at: DateTime, + pub downloads: Option, } #[derive(sqlx::FromRow, Debug)] @@ -46,6 +66,7 @@ pub struct LoaderVersionGetOne { pub win: Option, pub android: Option, pub ios: Option, + pub downloads: Option>, } pub struct GetVersionsQuery { @@ -72,6 +93,7 @@ impl LoaderVersionGetOne { android64: self.android, ios: self.ios, }, + downloads: self.downloads.map(|x| x.0), } } } @@ -86,7 +108,7 @@ impl LoaderVersion { ) -> Result, DatabaseError> { let mut query_builder: QueryBuilder = QueryBuilder::new( r#"SELECT - mac, win, android, ios, tag, commit_hash, created_at, prerelease + mac, win, android, ios, tag, commit_hash, created_at, prerelease, downloads FROM geode_versions "#, ); @@ -139,7 +161,7 @@ impl LoaderVersion { query_builder.push(" AND prerelease=FALSE "); } - query_builder.push(" ORDER BY "); + query_builder.push(" AND downloads IS NOT NULL ORDER BY "); if gd.is_none() && let Some(p) = platform @@ -179,7 +201,7 @@ impl LoaderVersion { LoaderVersionGetOne, r#"SELECT mac as "mac: _", win as "win: _", android as "android: _", ios as "ios: _", - tag, created_at, commit_hash, prerelease + tag, created_at, commit_hash, prerelease, downloads as "downloads: Json" FROM geode_versions WHERE tag = $1"#, tag @@ -229,7 +251,7 @@ impl LoaderVersion { let mut query_builder = QueryBuilder::new( r#" SELECT - mac, win, android, ios, tag, created_at, commit_hash, prerelease + mac, win, android, ios, tag, created_at, commit_hash, prerelease, downloads FROM geode_versions "#, ); From 1127fda9389853a1746d6705676773bd1495a118 Mon Sep 17 00:00:00 2001 From: dankmeme01 <42031238+dankmeme01@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:46:53 +0200 Subject: [PATCH 2/4] update query cache --- ...c6b3d2dc488e99dfb37cbb6b4ac1a13f76d63.json | 15 +++++++++++ ...24d312f8616475717b3f6b1e50cd95d4f86b3.json | 26 +++++++++++++++++++ ...d338ffeaeac49f79453305453cd312d6f0f7.json} | 18 ++++++++++--- 3 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 .sqlx/query-25c1b409c7444cdbe01c4aa09fbc6b3d2dc488e99dfb37cbb6b4ac1a13f76d63.json create mode 100644 .sqlx/query-42c856db81d1581a8fe41e42fa424d312f8616475717b3f6b1e50cd95d4f86b3.json rename .sqlx/{query-94a9f2bc14d5a6c21eb7f638e5d70bcdc7112d9ad0d153aaad6bf7b00e27d74a.json => query-4887f58d2c76c3c0f8e4dace53e1d338ffeaeac49f79453305453cd312d6f0f7.json} (88%) diff --git a/.sqlx/query-25c1b409c7444cdbe01c4aa09fbc6b3d2dc488e99dfb37cbb6b4ac1a13f76d63.json b/.sqlx/query-25c1b409c7444cdbe01c4aa09fbc6b3d2dc488e99dfb37cbb6b4ac1a13f76d63.json new file mode 100644 index 00000000..10ed2220 --- /dev/null +++ b/.sqlx/query-25c1b409c7444cdbe01c4aa09fbc6b3d2dc488e99dfb37cbb6b4ac1a13f76d63.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE geode_versions SET downloads = $1 WHERE tag = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Text" + ] + }, + "nullable": [] + }, + "hash": "25c1b409c7444cdbe01c4aa09fbc6b3d2dc488e99dfb37cbb6b4ac1a13f76d63" +} diff --git a/.sqlx/query-42c856db81d1581a8fe41e42fa424d312f8616475717b3f6b1e50cd95d4f86b3.json b/.sqlx/query-42c856db81d1581a8fe41e42fa424d312f8616475717b3f6b1e50cd95d4f86b3.json new file mode 100644 index 00000000..e1074e2e --- /dev/null +++ b/.sqlx/query-42c856db81d1581a8fe41e42fa424d312f8616475717b3f6b1e50cd95d4f86b3.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT tag FROM geode_versions WHERE downloads IS NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag", + "type_info": "Text", + "origin": { + "Table": { + "table": "geode_versions", + "name": "tag" + } + } + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "42c856db81d1581a8fe41e42fa424d312f8616475717b3f6b1e50cd95d4f86b3" +} diff --git a/.sqlx/query-94a9f2bc14d5a6c21eb7f638e5d70bcdc7112d9ad0d153aaad6bf7b00e27d74a.json b/.sqlx/query-4887f58d2c76c3c0f8e4dace53e1d338ffeaeac49f79453305453cd312d6f0f7.json similarity index 88% rename from .sqlx/query-94a9f2bc14d5a6c21eb7f638e5d70bcdc7112d9ad0d153aaad6bf7b00e27d74a.json rename to .sqlx/query-4887f58d2c76c3c0f8e4dace53e1d338ffeaeac49f79453305453cd312d6f0f7.json index 3933afcb..3138dd86 100644 --- a/.sqlx/query-94a9f2bc14d5a6c21eb7f638e5d70bcdc7112d9ad0d153aaad6bf7b00e27d74a.json +++ b/.sqlx/query-4887f58d2c76c3c0f8e4dace53e1d338ffeaeac49f79453305453cd312d6f0f7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n\t\t\t\t mac as \"mac: _\", win as \"win: _\", android as \"android: _\", ios as \"ios: _\",\n\t\t\t\t tag, created_at, commit_hash, prerelease\n\t\t\t FROM geode_versions\n\t\t\t\t WHERE tag = $1", + "query": "SELECT\n\t\t\t\t mac as \"mac: _\", win as \"win: _\", android as \"android: _\", ios as \"ios: _\",\n\t\t\t\t tag, created_at, commit_hash, prerelease, downloads as \"downloads: Json\"\n\t\t\t FROM geode_versions\n\t\t\t\t WHERE tag = $1", "describe": { "columns": [ { @@ -178,6 +178,17 @@ "name": "prerelease" } } + }, + { + "ordinal": 8, + "name": "downloads: Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "geode_versions", + "name": "downloads" + } + } } ], "parameters": { @@ -193,8 +204,9 @@ false, false, false, - false + false, + true ] }, - "hash": "94a9f2bc14d5a6c21eb7f638e5d70bcdc7112d9ad0d153aaad6bf7b00e27d74a" + "hash": "4887f58d2c76c3c0f8e4dace53e1d338ffeaeac49f79453305453cd312d6f0f7" } From 631bbe9cf3c153d4534556f5a175305d682ba95f Mon Sep 17 00:00:00 2001 From: dankmeme01 <42031238+dankmeme01@users.noreply.github.com> Date: Sat, 8 Aug 2026 21:24:45 +0200 Subject: [PATCH 3/4] redo to use tables --- ...c6b3d2dc488e99dfb37cbb6b4ac1a13f76d63.json | 15 -- ...24d312f8616475717b3f6b1e50cd95d4f86b3.json | 26 --- ...6e9538eb43ef12c45ac7e9c6f0d6314134b9.json} | 22 ++- ...dd_download_link_to_geode_version.down.sql | 6 +- ..._add_download_link_to_geode_version.up.sql | 12 +- src/database/repository/geode_versions.rs | 40 +++- src/s3_worker.rs | 153 ++++++++------- src/types/models/loader_version.rs | 178 +++++++++++++++--- 8 files changed, 307 insertions(+), 145 deletions(-) delete mode 100644 .sqlx/query-25c1b409c7444cdbe01c4aa09fbc6b3d2dc488e99dfb37cbb6b4ac1a13f76d63.json delete mode 100644 .sqlx/query-42c856db81d1581a8fe41e42fa424d312f8616475717b3f6b1e50cd95d4f86b3.json rename .sqlx/{query-4887f58d2c76c3c0f8e4dace53e1d338ffeaeac49f79453305453cd312d6f0f7.json => query-fa1aa2f18dc9286a9d2f9072d0ef6e9538eb43ef12c45ac7e9c6f0d6314134b9.json} (88%) diff --git a/.sqlx/query-25c1b409c7444cdbe01c4aa09fbc6b3d2dc488e99dfb37cbb6b4ac1a13f76d63.json b/.sqlx/query-25c1b409c7444cdbe01c4aa09fbc6b3d2dc488e99dfb37cbb6b4ac1a13f76d63.json deleted file mode 100644 index 10ed2220..00000000 --- a/.sqlx/query-25c1b409c7444cdbe01c4aa09fbc6b3d2dc488e99dfb37cbb6b4ac1a13f76d63.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE geode_versions SET downloads = $1 WHERE tag = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Text" - ] - }, - "nullable": [] - }, - "hash": "25c1b409c7444cdbe01c4aa09fbc6b3d2dc488e99dfb37cbb6b4ac1a13f76d63" -} diff --git a/.sqlx/query-42c856db81d1581a8fe41e42fa424d312f8616475717b3f6b1e50cd95d4f86b3.json b/.sqlx/query-42c856db81d1581a8fe41e42fa424d312f8616475717b3f6b1e50cd95d4f86b3.json deleted file mode 100644 index e1074e2e..00000000 --- a/.sqlx/query-42c856db81d1581a8fe41e42fa424d312f8616475717b3f6b1e50cd95d4f86b3.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT tag FROM geode_versions WHERE downloads IS NULL", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tag", - "type_info": "Text", - "origin": { - "Table": { - "table": "geode_versions", - "name": "tag" - } - } - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false - ] - }, - "hash": "42c856db81d1581a8fe41e42fa424d312f8616475717b3f6b1e50cd95d4f86b3" -} diff --git a/.sqlx/query-4887f58d2c76c3c0f8e4dace53e1d338ffeaeac49f79453305453cd312d6f0f7.json b/.sqlx/query-fa1aa2f18dc9286a9d2f9072d0ef6e9538eb43ef12c45ac7e9c6f0d6314134b9.json similarity index 88% rename from .sqlx/query-4887f58d2c76c3c0f8e4dace53e1d338ffeaeac49f79453305453cd312d6f0f7.json rename to .sqlx/query-fa1aa2f18dc9286a9d2f9072d0ef6e9538eb43ef12c45ac7e9c6f0d6314134b9.json index 3138dd86..53467e32 100644 --- a/.sqlx/query-4887f58d2c76c3c0f8e4dace53e1d338ffeaeac49f79453305453cd312d6f0f7.json +++ b/.sqlx/query-fa1aa2f18dc9286a9d2f9072d0ef6e9538eb43ef12c45ac7e9c6f0d6314134b9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n\t\t\t\t mac as \"mac: _\", win as \"win: _\", android as \"android: _\", ios as \"ios: _\",\n\t\t\t\t tag, created_at, commit_hash, prerelease, downloads as \"downloads: Json\"\n\t\t\t FROM geode_versions\n\t\t\t\t WHERE tag = $1", + "query": "SELECT\n\t\t\t\t mac as \"mac: _\", win as \"win: _\", android as \"android: _\", ios as \"ios: _\",\n\t\t\t\t tag, created_at, commit_hash, prerelease, resources_url, resources_hash\n\t\t\t FROM geode_versions\n\t\t\t\t WHERE tag = $1", "describe": { "columns": [ { @@ -181,12 +181,23 @@ }, { "ordinal": 8, - "name": "downloads: Json", - "type_info": "Jsonb", + "name": "resources_url", + "type_info": "Text", + "origin": { + "Table": { + "table": "geode_versions", + "name": "resources_url" + } + } + }, + { + "ordinal": 9, + "name": "resources_hash", + "type_info": "Text", "origin": { "Table": { "table": "geode_versions", - "name": "downloads" + "name": "resources_hash" } } } @@ -205,8 +216,9 @@ false, false, false, + true, true ] }, - "hash": "4887f58d2c76c3c0f8e4dace53e1d338ffeaeac49f79453305453cd312d6f0f7" + "hash": "fa1aa2f18dc9286a9d2f9072d0ef6e9538eb43ef12c45ac7e9c6f0d6314134b9" } diff --git a/migrations/20260721182316_add_download_link_to_geode_version.down.sql b/migrations/20260721182316_add_download_link_to_geode_version.down.sql index 52d0dc5a..8aa986d2 100644 --- a/migrations/20260721182316_add_download_link_to_geode_version.down.sql +++ b/migrations/20260721182316_add_download_link_to_geode_version.down.sql @@ -1 +1,5 @@ -ALTER TABLE geode_versions DROP COLUMN downloads; +DROP TABLE IF EXISTS geode_version_download; + +ALTER TABLE geode_versions + DROP COLUMN resources_url, + DROP COLUMN resources_hash; diff --git a/migrations/20260721182316_add_download_link_to_geode_version.up.sql b/migrations/20260721182316_add_download_link_to_geode_version.up.sql index cf96befd..1e62f4e6 100644 --- a/migrations/20260721182316_add_download_link_to_geode_version.up.sql +++ b/migrations/20260721182316_add_download_link_to_geode_version.up.sql @@ -1 +1,11 @@ -ALTER TABLE geode_versions ADD COLUMN downloads JSONB; +CREATE TABLE geode_version_download ( + tag TEXT NOT NULL REFERENCES geode_versions(tag) ON DELETE CASCADE, + platform TEXT NOT NULL, + url TEXT NOT NULL, + hash TEXT NOT NULL, + PRIMARY KEY (tag, platform) +); + +ALTER TABLE geode_versions + ADD COLUMN resources_url TEXT, + ADD COLUMN resources_hash TEXT; diff --git a/src/database/repository/geode_versions.rs b/src/database/repository/geode_versions.rs index 014853b8..934634cf 100644 --- a/src/database/repository/geode_versions.rs +++ b/src/database/repository/geode_versions.rs @@ -1,22 +1,44 @@ use sqlx::PgConnection; -use crate::{database::DatabaseError, types::models::loader_version::LoaderDownloads}; +use crate::database::DatabaseError; -#[tracing::instrument(skip_all, fields(tag = %tag, downloads = ?downloads))] -pub async fn update_downloads( +#[tracing::instrument(skip_all, fields(tag = %tag, platform = %platform, url = %url, hash = %hash))] +pub async fn upsert_download( tag: &str, - downloads: &LoaderDownloads, + platform: &str, + url: &str, + hash: &str, conn: &mut PgConnection, ) -> Result<(), DatabaseError> { - sqlx::query!( - "UPDATE geode_versions SET downloads = $1 WHERE tag = $2", - // should never fail - &serde_json::to_value(downloads).expect("serialization failed"), - tag + sqlx::query( + "INSERT INTO geode_version_download (tag, platform, url, hash) VALUES ($1, $2, $3, $4) + ON CONFLICT (tag, platform) DO UPDATE SET url = $3, hash = $4", ) + .bind(tag) + .bind(platform) + .bind(url) + .bind(hash) .execute(&mut *conn) .await .inspect_err(|e| tracing::error!("{:?}", e))?; Ok(()) } + +#[tracing::instrument(skip_all, fields(tag = %tag, url = %url, hash = %hash))] +pub async fn update_resources_download( + tag: &str, + url: &str, + hash: &str, + conn: &mut PgConnection, +) -> Result<(), DatabaseError> { + sqlx::query("UPDATE geode_versions SET resources_url = $2, resources_hash = $3 WHERE tag = $1") + .bind(tag) + .bind(url) + .bind(hash) + .execute(&mut *conn) + .await + .inspect_err(|e| tracing::error!("{:?}", e))?; + + Ok(()) +} diff --git a/src/s3_worker.rs b/src/s3_worker.rs index cafe15c0..c75eb9e3 100644 --- a/src/s3_worker.rs +++ b/src/s3_worker.rs @@ -2,14 +2,19 @@ use std::time::Duration; use actix_web::web; use bytes::Bytes; +use reqwest::StatusCode; use crate::{ config::AppData, database::repository::{ - geode_versions::update_downloads, mod_versions::update_managed_download_link, + geode_versions::{update_resources_download, upsert_download}, + mod_versions::update_managed_download_link, }, mod_zip, - types::models::{loader_version::LoaderDownloads, mod_gd_version::GDVersionEnum}, + types::models::{ + loader_version::{LoaderDownload, LoaderDownloads}, + mod_gd_version::GDVersionEnum, + }, }; pub enum S3WorkerTask { @@ -47,6 +52,59 @@ fn github_url_for_resources(tag: &str) -> String { format!("https://github.com/geode-sdk/geode/releases/download/v{tag}/resources.zip") } +async fn migrate_geode_version_opt( + data: &AppData, + tag: &str, + platform: &str, +) -> anyhow::Result> { + let storage = data.cdn_storage().expect("mod storage must be set by now"); + let (github_url, new_path) = match platform { + "resources" => (github_url_for_resources(tag), path_for_resources(tag)), + _ => ( + github_url_for_loader(tag, platform), + path_for_loader(tag, platform), + ), + }; + + let resp = data.http_client().get(&github_url).send().await?; + + if resp.status() == StatusCode::NOT_FOUND { + return Ok(None); + } + + let bytes = resp.error_for_status()?.bytes().await?; + let public_url = storage.asset_url(&new_path); + let hash = sha256::digest(&bytes[..]); + storage.store(&new_path, &bytes).await?; + + let mut conn = data.db().acquire().await?; + if platform == "resources" { + update_resources_download(tag, &public_url, &hash, &mut conn).await?; + } else { + upsert_download(tag, platform, &public_url, &hash, &mut conn).await?; + } + + Ok(Some(LoaderDownload { + url: public_url, + hash, + })) +} + +async fn migrate_geode_version( + data: &AppData, + tag: &str, + platform: &str, +) -> anyhow::Result { + match migrate_geode_version_opt(data, tag, platform).await? { + Some(download) => Ok(download), + None => Err(anyhow::anyhow!( + "Geode version {} for platform {} not found on GitHub", + tag, + platform + )), + } +} + async fn process_task(data: &AppData, task: S3WorkerTask) -> anyhow::Result<()> { let storage = data.cdn_storage().expect("mod storage must be set by now"); @@ -77,58 +135,25 @@ async fn process_task(data: &AppData, task: S3WorkerTask) -> anyhow::Result<()> S3WorkerTask::UploadLoader { tag } => { tracing::info!("Preparing to upload Geode v{tag} to S3"); - let mut zips = Vec::new(); - let mut downloads = LoaderDownloads::default(); - - zips.push(( - &mut downloads.win, - path_for_loader(&tag, "win"), - github_url_for_loader(&tag, "win"), - )); - zips.push(( - &mut downloads.mac, - path_for_loader(&tag, "mac"), - github_url_for_loader(&tag, "mac"), - )); - zips.push(( - &mut downloads.android32, - path_for_loader(&tag, "android32"), - github_url_for_loader(&tag, "android32"), - )); - zips.push(( - &mut downloads.android64, - path_for_loader(&tag, "android64"), - github_url_for_loader(&tag, "android64"), - )); - zips.push(( - &mut downloads.ios, - path_for_loader(&tag, "ios"), - github_url_for_loader(&tag, "ios"), - )); - zips.push(( - &mut downloads.resources, - path_for_resources(&tag), - github_url_for_resources(&tag), - )); - - for (dl, path, url) in zips { - let resp = data.http_client().get(&url).send().await?; - if let Err(e) = resp.error_for_status_ref() { - tracing::error!("Failed to download loader release from {url}: {e:?}",); - continue; + let ios = match migrate_geode_version_opt(data, &tag, "ios").await? { + Some(download) => download, + None => { + tracing::warn!( + "Geode version {} for iOS not found on GitHub, skipping", + tag + ); + LoaderDownload::default() } + }; - let resp = resp.bytes().await?; - let public_url = storage.asset_url(&path); - storage.store(&path, &resp).await?; - - dl.url = public_url; - dl.hash = sha256::digest(&resp[..]); - } - - let mut tx = data.db().begin().await?; - update_downloads(&tag, &downloads, &mut tx).await?; - tx.commit().await?; + let downloads = LoaderDownloads { + win: migrate_geode_version(data, &tag, "win").await?, + mac: migrate_geode_version(data, &tag, "mac").await?, + android32: migrate_geode_version(data, &tag, "android32").await?, + android64: migrate_geode_version(data, &tag, "android64").await?, + ios, + resources: migrate_geode_version(data, &tag, "resources").await?, + }; tracing::info!("Uploaded new loader release to S3: {downloads:?}"); } @@ -265,22 +290,20 @@ async fn migrate_existing_mods_to_s3(data: &AppData) -> anyhow::Result<()> { async fn migrate_loader_versions_to_s3(data: &AppData) -> anyhow::Result<()> { let mut db = data.db().acquire().await?; - let versions = sqlx::query!("SELECT tag FROM geode_versions WHERE downloads IS NULL") - .fetch_all(&mut *db) - .await?; + let versions: Vec = sqlx::query_scalar( + "SELECT gv.tag FROM geode_versions gv WHERE NOT EXISTS ( + SELECT 1 FROM geode_version_download gvd WHERE gvd.tag = gv.tag + )", + ) + .fetch_all(&mut *db) + .await + .map_err(|e| anyhow::anyhow!("{e}"))?; tracing::info!("Migrating {} Geode releases to S3", versions.len()); - for record in versions { - if let Err(e) = process_task( - data, - S3WorkerTask::UploadLoader { - tag: record.tag.clone(), - }, - ) - .await - { - tracing::error!("error migrating Geode release {} to S3: {e:?}", record.tag); + for tag in versions { + if let Err(e) = process_task(data, S3WorkerTask::UploadLoader { tag: tag.clone() }).await { + tracing::error!("error migrating Geode release {} to S3: {e:?}", tag); } } diff --git a/src/types/models/loader_version.rs b/src/types/models/loader_version.rs index aaf4da63..43c71a3c 100644 --- a/src/types/models/loader_version.rs +++ b/src/types/models/loader_version.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use crate::{ database::DatabaseError, types::{ @@ -11,10 +13,7 @@ use utoipa::ToSchema; use sqlx::{ PgConnection, Postgres, QueryBuilder, - types::{ - Json, - chrono::{DateTime, Utc}, - }, + types::chrono::{DateTime, Utc}, }; #[derive(Debug)] @@ -53,7 +52,7 @@ pub struct LoaderVersion { pub commit_hash: String, #[serde(with = "chrono_dt_secs")] pub created_at: DateTime, - pub downloads: Option, + pub downloads: LoaderDownloads, } #[derive(sqlx::FromRow, Debug)] @@ -66,7 +65,17 @@ pub struct LoaderVersionGetOne { pub win: Option, pub android: Option, pub ios: Option, - pub downloads: Option>, + + pub resources_url: Option, + pub resources_hash: Option, +} + +#[derive(sqlx::FromRow, Debug)] +pub struct GeodeVersionDownload { + pub tag: String, + pub platform: String, + pub url: String, + pub hash: String, } pub struct GetVersionsQuery { @@ -75,14 +84,86 @@ pub struct GetVersionsQuery { pub prerelease: bool, } +fn github_url(tag: &str, platform: &str) -> String { + format!( + "https://github.com/geode-sdk/geode/releases/download/v{tag}/geode-v{tag}-{platform}.zip" + ) +} + +fn github_resources_url(tag: &str) -> String { + format!("https://github.com/geode-sdk/geode/releases/download/v{tag}/resources.zip") +} + +impl LoaderDownload { + pub fn new_github(tag: &str, platform: &str) -> Self { + // this should only be called for versions that weren't migrated to S3 yet, + // temporarily serve the GitHub URLs and tell the client to not verify hashes + LoaderDownload { + url: github_url(tag, platform), + hash: String::new(), + } + } + + pub fn new_github_resources(tag: &str) -> Self { + LoaderDownload { + url: github_resources_url(tag), + hash: String::new(), + } + } +} + +fn build_downloads( + version: &LoaderVersionGetOne, + managed: Vec, +) -> LoaderDownloads { + let mut out = LoaderDownloads { + win: LoaderDownload::new_github(&version.tag, "windows"), + mac: LoaderDownload::new_github(&version.tag, "macos"), + android32: LoaderDownload::new_github(&version.tag, "android32"), + android64: LoaderDownload::new_github(&version.tag, "android64"), + ios: LoaderDownload::new_github(&version.tag, "ios"), + resources: LoaderDownload::new_github_resources(&version.tag), + }; + + for d in managed { + let download = LoaderDownload { + url: d.url, + hash: d.hash, + }; + + match d.platform.as_str() { + "win" => out.win = download, + "mac" => out.mac = download, + "android32" => out.android32 = download, + "android64" => out.android64 = download, + "ios" => out.ios = download, + _ => {} + } + } + + if let Some(url) = version.resources_url.clone() + && let Some(hash) = version.resources_hash.clone() + { + out.resources = LoaderDownload { url, hash }; + } + + out +} + impl LoaderVersionGetOne { - pub fn into_loader_version(self) -> LoaderVersion { + pub fn into_loader_version( + self, + managed_downloads: Vec, + ) -> LoaderVersion { + let downloads = build_downloads(&self, managed_downloads); + LoaderVersion { tag: format!("v{}", self.tag), version: self.tag, prerelease: self.prerelease, created_at: self.created_at, commit_hash: self.commit_hash, + downloads, gd: DetailedGDVersion { win: self.win, mac: self.mac, @@ -93,12 +174,45 @@ impl LoaderVersionGetOne { android64: self.android, ios: self.ios, }, - downloads: self.downloads.map(|x| x.0), } } } impl LoaderVersion { + pub async fn get_downloads_for_tag( + tag: &str, + pool: &mut PgConnection, + ) -> Result, DatabaseError> { + Ok(sqlx::query_as::<_, GeodeVersionDownload>( + "SELECT tag, platform, url, hash FROM geode_version_download WHERE tag = $1", + ) + .bind(tag) + .fetch_all(&mut *pool) + .await?) + } + + pub async fn get_downloads_for_tags( + tags: &[String], + pool: &mut PgConnection, + ) -> Result>, DatabaseError> { + if tags.is_empty() { + return Ok(HashMap::new()); + } + + let rows = sqlx::query_as::<_, GeodeVersionDownload>( + "SELECT tag, platform, url, hash FROM geode_version_download WHERE tag = ANY($1)", + ) + .bind(tags) + .fetch_all(&mut *pool) + .await?; + + let mut map = HashMap::<_, Vec<_>>::with_capacity(rows.len()); + for row in rows { + map.entry(row.tag.clone()).or_default().push(row); + } + Ok(map) + } + #[tracing::instrument(skip_all, fields(gd = ?gd, platform = ?platform, accept_prereleases = %accept_prereleases))] pub async fn get_latest( gd: Option, @@ -108,7 +222,7 @@ impl LoaderVersion { ) -> Result, DatabaseError> { let mut query_builder: QueryBuilder = QueryBuilder::new( r#"SELECT - mac, win, android, ios, tag, commit_hash, created_at, prerelease, downloads + mac, win, android, ios, tag, commit_hash, created_at, prerelease, resources_url, resources_hash FROM geode_versions "#, ); @@ -161,7 +275,7 @@ impl LoaderVersion { query_builder.push(" AND prerelease=FALSE "); } - query_builder.push(" AND downloads IS NOT NULL ORDER BY "); + query_builder.push(" ORDER BY "); if gd.is_none() && let Some(p) = platform @@ -183,13 +297,17 @@ impl LoaderVersion { query_builder.push(" created_at DESC LIMIT 1;"); - query_builder + let Some(row) = query_builder .build_query_as::() .fetch_optional(&mut *pool) .await - .inspect_err(|e| tracing::error!("{:?}", e)) - .map_err(|e| e.into()) - .map(|x| x.map(|y| y.into_loader_version())) + .inspect_err(|e| tracing::error!("{:?}", e))? + else { + return Ok(None); + }; + + let downloads = LoaderVersion::get_downloads_for_tag(&row.tag, &mut *pool).await?; + Ok(Some(row.into_loader_version(downloads))) } #[tracing::instrument(skip_all, fields(tag = %tag))] @@ -197,20 +315,24 @@ impl LoaderVersion { tag: &str, pool: &mut PgConnection, ) -> Result, DatabaseError> { - sqlx::query_as!( + let Some(row) = sqlx::query_as!( LoaderVersionGetOne, r#"SELECT mac as "mac: _", win as "win: _", android as "android: _", ios as "ios: _", - tag, created_at, commit_hash, prerelease, downloads as "downloads: Json" + tag, created_at, commit_hash, prerelease, resources_url, resources_hash FROM geode_versions WHERE tag = $1"#, tag ) .fetch_optional(&mut *pool) .await - .inspect_err(|e| tracing::error!("{:?}", e)) - .map_err(|e| e.into()) - .map(|x| x.map(|y| y.into_loader_version())) + .inspect_err(|e| tracing::error!("{:?}", e))? + else { + return Ok(None); + }; + + let downloads = LoaderVersion::get_downloads_for_tag(&row.tag, &mut *pool).await?; + Ok(Some(row.into_loader_version(downloads))) } #[tracing::instrument(skip_all, fields(tag = %version.tag))] @@ -251,7 +373,7 @@ impl LoaderVersion { let mut query_builder = QueryBuilder::new( r#" SELECT - mac, win, android, ios, tag, created_at, commit_hash, prerelease, downloads + mac, win, android, ios, tag, created_at, commit_hash, prerelease, resources_url, resources_hash FROM geode_versions "#, ); @@ -312,12 +434,22 @@ impl LoaderVersion { query_builder.push(" OFFSET "); query_builder.push_bind(offset); - query_builder + let rows = query_builder .build_query_as::() .fetch_all(&mut *pool) .await .inspect_err(|e| tracing::error!("{:?}", e)) - .map(|x| x.into_iter().map(|y| y.into_loader_version()).collect()) - .map_err(|e| e.into()) + .map_err(DatabaseError::from)?; + + let tags: Vec = rows.iter().map(|r| r.tag.clone()).collect(); + let mut downloads_map = LoaderVersion::get_downloads_for_tags(&tags, &mut *pool).await?; + + Ok(rows + .into_iter() + .map(|row| { + let downloads = downloads_map.remove(&row.tag).unwrap_or_default(); + row.into_loader_version(downloads) + }) + .collect()) } } From 6a7eeeef435dd72fd4a1665f5a837810cb4c521b Mon Sep 17 00:00:00 2001 From: dankmeme01 <42031238+dankmeme01@users.noreply.github.com> Date: Sat, 8 Aug 2026 21:39:33 +0200 Subject: [PATCH 4/4] no partial version skipping --- src/endpoints/loader.rs | 2 +- src/s3_worker.rs | 29 +++++++++++++++++------------ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/endpoints/loader.rs b/src/endpoints/loader.rs index 88e9b207..4237bceb 100644 --- a/src/endpoints/loader.rs +++ b/src/endpoints/loader.rs @@ -1,5 +1,5 @@ use actix_web::{HttpResponse, Responder, get, post, web}; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use std::str::FromStr; use utoipa::{IntoParams, ToSchema}; diff --git a/src/s3_worker.rs b/src/s3_worker.rs index c75eb9e3..750e04a2 100644 --- a/src/s3_worker.rs +++ b/src/s3_worker.rs @@ -54,6 +54,7 @@ fn github_url_for_resources(tag: &str) -> String { async fn migrate_geode_version_opt( data: &AppData, + db: &mut sqlx::Transaction<'_, sqlx::Postgres>, tag: &str, platform: &str, ) -> anyhow::Result> { @@ -77,11 +78,10 @@ async fn migrate_geode_version_opt( let hash = sha256::digest(&bytes[..]); storage.store(&new_path, &bytes).await?; - let mut conn = data.db().acquire().await?; if platform == "resources" { - update_resources_download(tag, &public_url, &hash, &mut conn).await?; + update_resources_download(tag, &public_url, &hash, db).await?; } else { - upsert_download(tag, platform, &public_url, &hash, &mut conn).await?; + upsert_download(tag, platform, &public_url, &hash, db).await?; } Ok(Some(LoaderDownload { @@ -92,13 +92,14 @@ async fn migrate_geode_version_opt( async fn migrate_geode_version( data: &AppData, + db: &mut sqlx::Transaction<'_, sqlx::Postgres>, tag: &str, platform: &str, ) -> anyhow::Result { - match migrate_geode_version_opt(data, tag, platform).await? { + match migrate_geode_version_opt(data, db, tag, platform).await? { Some(download) => Ok(download), None => Err(anyhow::anyhow!( - "Geode version {} for platform {} not found on GitHub", + "Geode version {} for platform '{}' not found on GitHub", tag, platform )), @@ -135,11 +136,13 @@ async fn process_task(data: &AppData, task: S3WorkerTask) -> anyhow::Result<()> S3WorkerTask::UploadLoader { tag } => { tracing::info!("Preparing to upload Geode v{tag} to S3"); - let ios = match migrate_geode_version_opt(data, &tag, "ios").await? { + let mut tx = data.db().begin().await?; + + let ios = match migrate_geode_version_opt(data, &mut tx, &tag, "ios").await? { Some(download) => download, None => { tracing::warn!( - "Geode version {} for iOS not found on GitHub, skipping", + "Geode version {} for iOS not found on GitHub, skipping iOS", tag ); LoaderDownload::default() @@ -147,14 +150,16 @@ async fn process_task(data: &AppData, task: S3WorkerTask) -> anyhow::Result<()> }; let downloads = LoaderDownloads { - win: migrate_geode_version(data, &tag, "win").await?, - mac: migrate_geode_version(data, &tag, "mac").await?, - android32: migrate_geode_version(data, &tag, "android32").await?, - android64: migrate_geode_version(data, &tag, "android64").await?, + win: migrate_geode_version(data, &mut tx, &tag, "win").await?, + mac: migrate_geode_version(data, &mut tx, &tag, "mac").await?, + android32: migrate_geode_version(data, &mut tx, &tag, "android32").await?, + android64: migrate_geode_version(data, &mut tx, &tag, "android64").await?, ios, - resources: migrate_geode_version(data, &tag, "resources").await?, + resources: migrate_geode_version(data, &mut tx, &tag, "resources").await?, }; + tx.commit().await?; + tracing::info!("Uploaded new loader release to S3: {downloads:?}"); } }