-
-
-
-
-
+
+
@@ -121,7 +124,7 @@
Code
Datasets
Models
-
Spaces
+
Demos
diff --git a/main.js b/main.js
index bf6c297..d47902b 100644
--- a/main.js
+++ b/main.js
@@ -6,12 +6,15 @@
// SECTION 1: CONFIGURATION AND STATE MANAGEMENT
//
-import jsYaml from 'js-yaml';
-import { normalizeTag } from './src/normalizeTag.js';
-import { getPlatformApiUrls } from './src/defineApiUrls.js';
-import { getPlatformDisplay } from './src/defineRibbonVals.js';
-import { filterItems, sortItems } from './src/filterAndSort.js';
-import { filterNewAdditionalEntries } from './src/filterNewAdditionalEntries.js';
+import { load } from 'js-yaml';
+import { initializeUIFromConfig, setThemeToggle } from './src/ui/initUserInterface.js';
+import { parseUrlParams, updateUrlParams, getCurrentState } from './src/ui/urlManager.js';
+import { renderItemList } from './src/ui/render.js';
+import { getPlatformApiUrls } from './src/utils/definePlatformVals.js';
+import { filterItems, sortItems } from './src/utils/filterAndSort.js';
+import { fetchCodeRepos } from './src/api/fetchCodeRepos.js';
+import { fetchHfRepos } from './src/api/fetchHfRepos.js';
+import { fetchCatalogStats } from './src/api/fetchStats.js';
// Start fetching config immediately when the module loads (before DOMContentLoaded)
// so the fetch is in-flight while the DOM is being parsed.
@@ -20,29 +23,12 @@ const configPromise = fetch('config.yaml')
if (!r.ok) throw new Error(`Failed to load config.yaml: HTTP ${r.status}`);
return r.text();
})
- .then(text => jsYaml.load(text));
+ .then(text => load(text));
// Module-scope lets ā assigned after config loads, used by all functions below
let CONFIG;
-let ORGANIZATION_NAME, CATALOG_REPO_NAME, PLATFORM, API_BASE_URL, REFRESH_INTERVAL_DAYS, ADDITIONAL_REPOS, ADDITIONAL_HF_REPOS;
-let ORG_API_URL, REPO_API_URL;
-
-// Build a reverse lookup from TAG_GROUPS (defined in tag-groups.js): raw tag ā [canonical tags]
-// A raw tag may appear in multiple groups, so the value is an array.
-const tagLookup = Object.create(null);
-if (typeof TAG_GROUPS !== 'undefined') {
- for (const [canonical, aliases] of Object.entries(TAG_GROUPS)) {
- for (const alias of aliases) {
- const key = alias.toLowerCase();
- if (tagLookup[key]) {
- tagLookup[key].push(canonical);
- } else {
- tagLookup[key] = [canonical];
- }
- }
- }
-}
-
+let ORGANIZATION_NAME, HF_ORGANIZATION_NAME, CATALOG_REPO_NAME, PLATFORM, API_BASE_URL, REFRESH_INTERVAL_DAYS, ADDITIONAL_REPOS, ADDITIONAL_HF_REPOS;
+let ORG_API_URL, REPO_API_URL, RELEASE_SUFFIX;
let releasesMap = {};
@@ -65,103 +51,8 @@ let fetchedData = {
spaces: false
};
-// Utility function to handle errors and display a user-friendly message
-const handleError = (error, message) => {
- console.error(message, error);
- const itemList = document.getElementById('itemList');
- itemList.innerHTML = `
-
Error loading items.
-
${message}
-
`;
-};
-
-//
-// SECTION 1B: URL PARAMETER HANDLING
-//
-
-/**
- * Parses URL parameters from both query string (?key=value) and hash (#key=value).
- * Query parameters take precedence over hash parameters.
- * @returns {Object} An object containing the parsed parameters.
- */
-const parseUrlParams = () => {
- const params = {};
-
- // Parse query string parameters (e.g., ?type=datasets&q=fish)
- const queryString = window.location.search;
- const urlParams = new URLSearchParams(queryString);
-
- // Parse hash parameters (e.g., #type=datasets&q=fish)
- const hash = window.location.hash.slice(1); // Remove the leading '#'
- const hashParams = new URLSearchParams(hash);
-
- // Hash parameters first (lower precedence)
- for (const [key, value] of hashParams) {
- params[key] = value;
- }
-
- // Query parameters override hash parameters (higher precedence)
- for (const [key, value] of urlParams) {
- params[key] = value;
- }
-
- return params;
-};
-
-/**
- * Updates the URL hash with the current filter state without triggering a page reload.
- * @param {Object} state - The current state object with type, q, sort, tag properties.
- */
-const updateUrlParams = (state) => {
- const params = new URLSearchParams();
-
- // Only add non-default values to the URL
- if (state.type && state.type !== 'all') {
- params.set('type', state.type);
- }
- if (state.q && state.q.trim() !== '') {
- params.set('q', state.q);
- }
- if (state.sort && state.sort !== 'lastModified') {
- params.set('sort', state.sort);
- }
- if (state.tag && state.tag !== '') {
- params.set('tag', state.tag);
- }
- if (state.archived && state.archived !== 'active') {
- params.set('archived', state.archived);
- }
-
- const paramString = params.toString();
- const newHash = paramString ? `#${paramString}` : '';
-
- // Build the new URL preserving the pathname and any query parameters when clearing hash
- const baseUrl = window.location.pathname + window.location.search;
- const newUrl = newHash ? baseUrl + newHash : baseUrl;
-
- // Update the URL without triggering a page reload
- const currentUrl = window.location.pathname + window.location.search + window.location.hash;
- if (currentUrl !== newUrl) {
- history.replaceState(null, '', newUrl);
- }
-};
-
-/**
- * Gets the current filter state from form elements.
- * @returns {Object} The current state object.
- */
-const getCurrentState = () => {
- return {
- type: document.getElementById('repoType')?.value || 'all',
- q: document.getElementById('searchInput')?.value || '',
- sort: document.getElementById('sortBy')?.value || 'lastModified',
- tag: document.getElementById('tagFilter')?.value || '',
- archived: document.getElementById('archiveFilter')?.value || 'active'
- };
-};
-
//
-// SECTION 2: DATA FETCHING LOGIC
+// SECTION 2: DATA FETCHING
//
/**
@@ -178,388 +69,47 @@ const fetchHubItems = async (repoType) => {
const skeletons = document.querySelectorAll('.skeleton-card');
skeletons.forEach(s => s.classList.remove('hidden'));
- try {
- let items = [];
-
- // Platform-specific api requests for code
- if (repoType === "code") {
- if (fetchedData.code) return allItems.code // reuse if already fetched
-
- // Paginate through all public repos (Platform determines API max returns per page)
- let allRepos = [];
- let nextUrl = `${ORG_API_URL}`;
- while (nextUrl) {
- const ghResponse = await fetch(nextUrl);
-
- if (!ghResponse.ok) {
- const platformDisplay = getPlatformDisplay(PLATFORM);
- throw new Error(`${platformDisplay.displayName || PLATFORM} error: ${ghResponse.status}`);
- }
-
- const page = await ghResponse.json();
- allRepos = allRepos.concat(page);
-
- // Parse the Link header to find the next page URL, if any
- const linkHeader = ghResponse.headers.get('Link');
- const match = linkHeader && linkHeader.match(/<([^>]+)>;\s*rel="next"/);
- nextUrl = match ? match[1] : null;
- }
-
- // For org-owned entries in ADDITIONAL_REPOS, reuse data already in allRepos to avoid redundant API calls.
- // Only fetch entries that belong to a different org (external repos).
- const allReposByFullName = new Map(allRepos.map(r => [r.full_name, r]));
- const toFetch = ADDITIONAL_REPOS.filter(ownerRepo => !allReposByFullName.has(ownerRepo));
- const fromAllRepos = ADDITIONAL_REPOS.map(ownerRepo => allReposByFullName.get(ownerRepo)).filter(Boolean);
-
- const fetchedExternalData = await Promise.all(
- toFetch.map(ownerRepo =>
- fetch(`${REPO_API_URL}${ownerRepo}`)
- .then(r => {
- if (!r.ok) {
- console.warn(`Failed to fetch additional repo "${ownerRepo}": HTTP ${r.status}`);
- return null;
- }
- return r.json();
- })
- .catch(err => {
- console.warn(`Network error fetching additional repo "${ownerRepo}":`, err);
- return null;
- })
- )
- );
- const additionalRepos = [...fromAllRepos, ...fetchedExternalData.filter(Boolean)];
-
- // Keep only non-forks from org; deduplicate against additional repos by full_name
- const orgRepoNames = new Set(additionalRepos.map(r => r.full_name));
- const orgNonForks = allRepos.filter(repo => repo.name !== ".github" && !repo.fork && !orgRepoNames.has(repo.full_name));
-
- // Process additional repos and all remaining org non-forks to include metadata and 'new' flag as appropriate
- items = [...additionalRepos, ...orgNonForks]
- .map(repo => {
- const createdAt = new Date(repo.created_at);
- const lastModified = new Date(repo.updated_at);
- const isNew = (new Date() - createdAt) / (1000 * 60 * 60 * 24) < REFRESH_INTERVAL_DAYS;
-
- const rawTags = (repo.topics || []).map(t => t.toLowerCase());
- const tags = [...new Set(rawTags.flatMap(t => normalizeTag(t, tagLookup)).filter(Boolean))];
- const displayTags = rawTags.filter(t => !t.includes(':'));
- tags.forEach(tag => tagsMap.code.add(tag));
-
- const release = releasesMap[repo.full_name] ?? null;
-
- return {
- id: repo.full_name, // "Imageomics/
", used as backup if can't get repo.name
- repoType: "code",
- createdAt,
- lastModified,
- isNew,
- archived: repo.archived || false,
- tags,
- rawTags,
- displayTags,
- description: repo.description || "No description provided.",
- html_url: repo.html_url,
- hasNewRelease: release?.isNew ?? false,
- latestReleaseUrl: release?.url ?? null,
- latestReleaseTag: release?.tag ?? null,
- cardData: {
- pretty_name: repo.name, // , the one used for card title display
- description: repo.description,
- stars: repo.stargazers_count
- }
- };
- });
-
- allItems.code = items;
- fetchedData.code = true;
-
- skeletons.forEach(s => s.classList.add('hidden'));
-
- return items;
- }
+ let items = []
- // hugging face api requests for datasets/models/spaces
- const response = await fetch(`${API_BASE_URL}${repoType}?author=${ORGANIZATION_NAME}&full=true`);
- if (!response.ok) {
- throw new Error(`HTTP error! status: ${response.status}`);
- }
- let hfItems = await response.json();
-
- // Fetch additional HF repos of this type from outside the org
- const additionalForType = ADDITIONAL_HF_REPOS.filter(entry => entry.type === repoType);
- if (additionalForType.length) {
- const existingIds = new Set(hfItems.map(item => item.id));
- const toFetch = filterNewAdditionalEntries(existingIds, additionalForType);
-
- const fetched = await Promise.all(
- toFetch.map(entry =>
- fetch(`${API_BASE_URL}${repoType}/${entry.repo}`)
- .then(r => {
- if (!r.ok) {
- console.warn(`Failed to fetch additional HF repo "${entry.repo}": HTTP ${r.status}`);
- return null;
- }
- return r.json();
- })
- .catch(err => {
- console.warn(`Network error fetching additional HF repo "${entry.repo}":`, err);
- return null;
- })
- )
- );
- hfItems = [...hfItems, ...fetched.filter(item => item && !existingIds.has(item.id))];
+ if (repoType === 'code') {
+ // Check that releasesMap is populated; if not, fetch it from releases.json
+ if (Object.keys(releasesMap).length === 0) {
+ releasesMap = await fetch('./releases.json')
+ .then(res => res.ok ? res.json() : {})
+ .catch(() => ({}));
}
-
- // Step 2: If we are fetching models, get the full details for each one.
- if (repoType === 'models') {
- const detailPromises = hfItems.map(item =>
- fetch(`${API_BASE_URL}models/${item.id}`).then(res => {
- if (!res.ok) {
- console.error(`Failed to fetch details for ${item.id}`);
- return null; // Return null for failed requests
- }
- return res.json();
- })
- );
-
- // Wait for all detail requests to complete in parallel.
- const detailedItems = await Promise.all(detailPromises);
-
- // Filter out any models that failed to fetch and assign the detailed list.
- hfItems = detailedItems.filter(Boolean);
- }
-
- // Process the data to include metadata and a 'new' flag
- const processedItems = hfItems.map(item => {
- const createdAt = new Date(item.createdAt);
- const lastModified = new Date(item.lastModified);
- const isNew = (new Date() - createdAt) / (1000 * 60 * 60 * 24) < REFRESH_INTERVAL_DAYS;
-
- // Extract tags from the YAML metadata (handling different structures)
- const rawTags = (item.cardData?.tags || item.tags || []).map(t => String(t).toLowerCase());
- const tags = [...new Set(rawTags.flatMap(t => normalizeTag(t, tagLookup)).filter(Boolean))];
- const displayTags = rawTags.filter(t => !t.includes(':'));
- tags.forEach(tag => tagsMap[repoType].add(tag));
-
- return {
- ...item,
- repoType,
- createdAt,
- lastModified,
- isNew,
- archived: false,
- likes: item.likes || 0,
- tags,
- rawTags,
- displayTags
- };
- });
-
- allItems[repoType] = processedItems;
- fetchedData[repoType] = true;
-
- skeletons.forEach(s => s.classList.add('hidden'));
-
- return processedItems;
- } catch (error) {
- handleError(error, `Failed to fetch ${repoType}. Please check your network connection or the API.`);
- return [];
- }
-};
-
-/**
- * Fetches statistics for the Catalog repository itself (Stars, Forks, Version)
- * populates the badge in the top right corner.
- */
-const fetchCatalogStats = async () => {
- // Helper: Updates text, shows the specific stat, and ensures the divider is visible
- const update = (textId, containerId, value) => {
- const el = document.getElementById(textId);
- const container = document.getElementById(containerId);
- if (el && container && value !== undefined) {
- el.innerText = value;
- if (value != 0) {
- container.classList.remove('hidden');
- container.classList.add('flex');
- }
- }
- };
-
- try {
- //TODO: Update stars and forks to support other platforms (GitLab, Codeberg) once implemented
- // 1. Get Stars & Forks
- const repo = await fetch(`${REPO_API_URL}${ORGANIZATION_NAME}/${CATALOG_REPO_NAME}`).then(r => r.ok ? r.json() : {});
- if (repo.stargazers_count !== undefined) update('gh-stars', 'gh-star-container', repo.stargazers_count);
- if (repo.forks_count !== undefined) update('gh-forks', 'gh-fork-container', repo.forks_count);
-
- // 2. Get Version (Tag)
- // TODO: Import from package.json
- const release = await fetch(`${REPO_API_URL}${ORGANIZATION_NAME}/${CATALOG_REPO_NAME}/releases/latest`).then(r => r.ok ? r.json() : {});
- if (release.tag_name) update('gh-tag', 'gh-version-container', release.tag_name);
-
- } catch (e) {
- console.warn("Could not fetch Code Repo stats", e);
- }
-};
-
-//
-// SECTION 3: RENDERING LOGIC
-//
-
-/**
- * Escapes HTML special characters in a string to prevent XSS.
- * @param {string} str - The input string.
- * @returns {string} The escaped string.
- */
-const escapeHTML = (str) => {
- if (!str) return "";
- return str.replace(/[&<>"']/g, (m) => ({
- '&': '&',
- '<': '<',
- '>': '>',
- '"': '"',
- "'": '''
- }[m]));
-};
-
-/**
- * Adds word break opportunities after underscores to allow proper text wrapping.
- * @param {string} str - The input string.
- * @returns {string} The string with tags inserted after underscores.
- */
-const addWordBreakOpportunities = (str) => {
- if (!str) return "";
- // Replace underscores with underscore + word break opportunity
- return str.replace(/_/g, '_');
-};
-
-/**
- * Renders a single item card (code, dataset, model, or space) to HTML.
- * @param {Object} item - The item object to render.
- * @param {string} repoType - The type of repository.
- * @returns {string} The HTML string for the item card.
- */
-const renderHubItemCard = (item, repoType) => {
- const lastUpdatedDate = new Date(item.lastModified).toLocaleDateString();
- const tagsHtml = (item.displayTags || item.rawTags || []).map(tag =>
- `${tag} `
- ).join('');
-
- // Use pretty_name for the heading, with a fallback
- // HF API keys for CardData: https://huggingface.co/docs/huggingface_hub/main/en/package_reference/cards#huggingface_hub.CardData
- // datasets have pretty_name, models have model_name, spaces have title
- const prettyName = item.cardData?.pretty_name || item.cardData?.model_name || item.cardData?.title || item.id.split('/')[1];
-
- // Use the description from cardData, with fallbacks
- const displayDescription = item.cardData?.description || item.cardData?.model_description || item.description || 'No description provided.';
-
- // Construct the correct URL based on the repository type
- let itemUrl;
-
- switch (item.repoType) {
- case "code":
- itemUrl = item.html_url;
- break;
- case "datasets":
- itemUrl = `https://huggingface.co/datasets/${item.id}`;
- break;
- case "spaces":
- itemUrl = `https://huggingface.co/spaces/${item.id}`;
- break;
- case "models":
- itemUrl = `https://huggingface.co/${item.id}`;
- break;
- default:
- // fallback for "all"
- itemUrl = `https://huggingface.co/${item.id}`;
- break;
+ items = await fetchCodeRepos(
+ PLATFORM,
+ ADDITIONAL_REPOS,
+ ORG_API_URL,
+ REPO_API_URL,
+ REFRESH_INTERVAL_DAYS,
+ releasesMap
+ );
+ } else {
+ items = await fetchHfRepos(
+ repoType,
+ ADDITIONAL_HF_REPOS,
+ API_BASE_URL,
+ HF_ORGANIZATION_NAME,
+ REFRESH_INTERVAL_DAYS
+ );
}
- // stars for code repos
- const badgeHtml = (() => {
- if (item.isNew) {
- return `
- New!
- `;
- }
-
- if (typeof item.cardData.stars === "number" && item.cardData.stars > 0) {
- return `
- ā ${item.cardData.stars}
- `;
- }
-
- if (typeof item.likes === "number" && item.likes > 0) {
- return `
-
- ā¤ļø ${item.likes}
- `;
- }
- return "";
- })();
-
- const escapedTitle = escapeHTML(prettyName);
- const displayTitle = addWordBreakOpportunities(escapedTitle);
-
- return `
-
-
-
-
- ${displayDescription}
-
-
-
-
- ${tagsHtml}
-
-
- Updated: ${lastUpdatedDate}
- ${item.archived ? `Archived ` : ''}
-
-
-
} - A promise resolving to an array of results
+ */
+async function mapConcurrent(items, concurrency, fn) {
+ const results = new Array(items.length);
+ let index = 0;
+ const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+ while (index < items.length) {
+ const i = index++;
+ results[i] = await fn(items[i]);
+ }
+ });
+ await Promise.all(workers);
+ return results;
+}
+
+/**
+ * Fetch the latest release for a given repo ID.
+ * @param {String} id - repo ID ('owner/repo' format)
+ * @returns {Promise<[String, Object|null]>} - A promise resolving to a tuple of [repo ID, release data or null if not found]
+ */
+async function fetchRepoRelease(id) {
try {
- const res = await fetch(`${REPO_API_URL}${id}/releases/latest`, { headers });
- if (!res.ok) { releases[id] = null; continue; }
+ const res = await fetch(`${REPO_API_URL}${encodeRepoId(id)}/${RELEASE_SUFFIX}`, { headers });
+ if (!res.ok) return [id, null];
+
const data = await res.json();
- releases[id] = {
+ const publishedAt = data[releasePublishedAtKey];
+ // gitlab's release API returns a different structure than GitHub/Codeberg
+ return [id, {
tag: data.tag_name,
- url: data.html_url,
- publishedAt: data.published_at,
- isNew: (Date.now() - new Date(data.published_at)) < TWO_WEEKS_MS,
- };
- } catch {
- releases[id] = null;
+ url: getReleaseUrl(data, id),
+ publishedAt: publishedAt,
+ isNew: (Date.now() - new Date(publishedAt)) < TWO_WEEKS_MS,
+ }];
+ } catch (err) {
+ console.warn(`[fetch-releases] Network error for repo "${id}":`, err.message);
+ return [id, null];
}
-}
+};
+
+// fetch the repo releases
+const releaseEntries = await mapConcurrent(repoIds, 5, fetchRepoRelease);
+
+// Convert [id, releaseData] pairs into an object for JSON output
+const releases = Object.fromEntries(releaseEntries);
writeFileSync(join(__dirname, '../public/releases.json'), JSON.stringify(releases));
console.log(`Wrote releases.json (${Object.keys(releases).length} repos)`);
diff --git a/scripts/platformScriptHelpers.js b/scripts/platformScriptHelpers.js
new file mode 100644
index 0000000..9354f32
--- /dev/null
+++ b/scripts/platformScriptHelpers.js
@@ -0,0 +1,73 @@
+/**
+ * Script utilities for platform-specific API calls and headers when exporting tags or fetching repo release data.
+ * Collection of functions to define platform-specific values needed only in build-time/workflow scripts.
+ * Each function returns dictionaries with the platform values that have been keyed by platform name (e.g., 'github', 'gitlab', 'codeberg') and potentially other instance-specific values (e.g., organizationName, repoApiUrl).
+ * Platform and organization name are defined from config.yaml and passed to the functions requiring them.
+*/
+
+
+/**
+ * Utility function to get platform-specific values for repo data keys (e.g., star count and profile repo name).
+ * @param {string} platform - 'github', 'gitlab', or 'codeberg'
+ * @returns {Object} platformVals - An object containing platform-specific values for repo data keys (e.g., release URL and published date key)
+ */
+export function getPlatformReleaseVals(platform) {
+ const platformVals = {
+ github: {
+ getReleaseUrl: (data) => data.html_url,
+ releasePublishedAtKey: 'published_at'
+ },
+ gitlab: {
+ getReleaseUrl: (data, repoId) => {
+ const tag = encodeURIComponent(data.tag_name);
+ return `https://gitlab.com/${repoId}/-/releases/${tag}`;
+ },
+ releasePublishedAtKey: 'released_at'
+ },
+ codeberg: {
+ getReleaseUrl: (data) => data.html_url,
+ releasePublishedAtKey: 'published_at'
+ }
+};
+ return platformVals[platform.toLowerCase()];
+}
+
+// Define platform-specific token authentication and API host information for scripts to use when making API calls.
+const tokenAuthByPlatform = {
+ github: {
+ token: process.env.GITHUB_TOKEN || process.env.GH_TOKEN,
+ apiHost : 'api.github.com',
+ authScheme: 'Bearer',
+ },
+ gitlab: {
+ token: process.env.GITLAB_TOKEN,
+ apiHost : 'gitlab.com/api/v4',
+ authScheme: 'Bearer',
+ },
+ codeberg: {
+ token: process.env.CODEBERG_TOKEN,
+ apiHost : 'codeberg.org/api/v1',
+ authScheme: 'token'
+ },
+};
+/**
+ * Utility function for scripts to get platform-specific headers
+* Update the corresponding workflow as needed for non-GitHub code platforms (e.g., Codeberg or GitLab)
+* `headers['Accept']` is only needed for GitHub to avoid 403 errors on some endpoints.
+* @param {string} url - URL to check for the platform API host
+* @param {string} platform - 'github', 'gitlab', or 'codeberg'
+* @param {string} orgName - The name of the organization as used in API calls
+* @returns {object} headers - headers object to use for fetch requests
+*/
+export function getPlatformHeaders(url, platform, orgName) {
+ const { token, apiHost, authScheme } = tokenAuthByPlatform[platform];
+ const cleanedOrgName = orgName.replace(/[^a-zA-Z0-9_-]/g, '');
+ const headers = { 'User-Agent': `${cleanedOrgName}-catalog-build-script` };
+ if (token && apiHost && url.includes(apiHost)) {
+ headers['Authorization'] = `${authScheme} ${token}`;
+ if (platform === 'github') {
+ headers['Accept'] = 'application/vnd.github+json';
+ }
+ }
+ return headers;
+}
diff --git a/src/api/fetchCodeRepos.js b/src/api/fetchCodeRepos.js
new file mode 100644
index 0000000..c918ac4
--- /dev/null
+++ b/src/api/fetchCodeRepos.js
@@ -0,0 +1,123 @@
+import { handleError } from '../ui/render.js';
+import { getPlatformVals } from '../utils/definePlatformVals.js';
+import { getPlatformDisplay } from '../utils/defineRibbonVals.js';
+import { normalizeTag, filterDisplayTags } from '../utils/normalizeTag.js';
+
+/**
+ * Function for fetching code repositories from the specified platform (GitHub, GitLab, or Codeberg).
+ * Both org-owned (non-forks) and additional repos are fetched; metadata is processed for each repo.
+ * It also determines if a repo is "new" based on the provided refresh interval.
+ * @async
+ * @param {string} platform - 'github', 'gitlab', or 'codeberg'
+ * @param {Array} additionalRepos - An array of additional "owner/repo" strings to include in addition to non-forked
+ * org repos.
+ * @param {string} orgApiUrl - The API URL for fetching organization repos
+ * @param {string} repoApiUrl - The API URL for fetching individual repo details
+ * @param {number} refreshIntervalDays - The cutoff in days for determining if a repo is "new"
+ * @param {Record} releasesMap -
+ * Release information for repos, keyed by full_name
+ * @returns {Promise} processedItems - A promise resolving to an array of code repositories
+ */
+export async function fetchCodeRepos(
+ platform,
+ additionalRepos,
+ orgApiUrl,
+ repoApiUrl,
+ refreshIntervalDays,
+ releasesMap
+) {
+
+ let allRepos = [];
+ let nextUrl = `${orgApiUrl}`;
+ // get platform-specific keys
+ const { starsKey, profileRepo, fullNameKey, updatedAtKey, forkKey, urlKey, encodeRepoId } = getPlatformVals(platform);
+ try {
+ while (nextUrl) {
+ const ghResponse = await fetch(nextUrl);
+
+ if (!ghResponse.ok) {
+ const platformDisplay = getPlatformDisplay(platform);
+ throw new Error(`${platformDisplay.displayName || platform} error: ${ghResponse.status}`);
+ }
+
+ const page = await ghResponse.json();
+ allRepos = allRepos.concat(page);
+
+ // Parse the Link header to find the next page URL, if any
+ const linkHeader = ghResponse.headers.get('Link');
+ const match = linkHeader && linkHeader.match(/<([^>]+)>;\s*rel="next"/);
+ nextUrl = match ? match[1] : null;
+ }
+
+ // For org-owned entries in additionalRepos, reuse data already in allRepos to avoid redundant API calls.
+ // Only fetch entries that belong to a different org (external repos).
+ const allReposByFullName = new Map(allRepos.map(r => [r[fullNameKey], r]));
+ const toFetch = additionalRepos.filter(ownerRepo => !allReposByFullName.has(ownerRepo));
+ const fromAllRepos = additionalRepos.map(ownerRepo => allReposByFullName.get(ownerRepo)).filter(Boolean);
+
+ const fetchedExternalData = await Promise.all(
+ toFetch.map(ownerRepo =>
+ fetch(`${repoApiUrl}${encodeRepoId(ownerRepo)}`)
+ .then(r => {
+ if (!r.ok) {
+ console.warn(`Failed to fetch additional repo "${ownerRepo}": HTTP ${r.status}`);
+ return null;
+ }
+ return r.json();
+ })
+ .catch(err => {
+ console.warn(`Network error fetching additional repo "${ownerRepo}":`, err);
+ return null;
+ })
+ )
+ );
+ const filteredAdditionalRepos = [...fromAllRepos, ...fetchedExternalData.filter(Boolean)];
+
+ // Keep only non-forks from org; deduplicate against additional repos by full_name
+ const orgRepoNames = new Set(filteredAdditionalRepos.map(r => r[fullNameKey]));
+ const orgNonForks = allRepos.filter(repo =>
+ repo.name !== profileRepo &&
+ !repo[forkKey] &&
+ !orgRepoNames.has(repo[fullNameKey]));
+
+ // Process additional repos and all remaining org non-forks to include metadata and 'new' flag as appropriate
+ let processedItems = [...filteredAdditionalRepos, ...orgNonForks]
+ .map(repo => {
+ const createdAt = new Date(repo.created_at);
+ const lastModified = new Date(repo[updatedAtKey]);
+ const isNew = (new Date() - createdAt) / (1000 * 60 * 60 * 24) < refreshIntervalDays;
+
+ const rawTags = (repo.topics || []).map(t => t.toLowerCase());
+ const tags = [...new Set(rawTags.flatMap(t => normalizeTag(t)).filter(Boolean))];
+ const displayTags = filterDisplayTags(rawTags);
+
+ const release = releasesMap[repo[fullNameKey]] ?? null;
+
+ return {
+ id: repo[fullNameKey], // "Imageomics/", used as backup if can't get repo.name
+ repoType: "code",
+ createdAt,
+ lastModified,
+ isNew,
+ archived: repo.archived || false,
+ tags,
+ rawTags,
+ displayTags,
+ description: repo.description, // fallback in display (render.js)
+ html_url: repo[urlKey],
+ hasNewRelease: release?.isNew ?? false,
+ latestReleaseUrl: release?.url ?? null,
+ latestReleaseTag: release?.tag ?? null,
+ cardData: {
+ pretty_name: repo.name, // , the one used for card title display
+ stars: repo[starsKey] ?? 0
+ }
+ };
+ });
+
+ return processedItems;
+ } catch (error) {
+ handleError(error, `Failed to fetch code from ${platform}. Please check your network connection or the API.`);
+ return [];
+ }
+}
diff --git a/src/api/fetchHfRepos.js b/src/api/fetchHfRepos.js
new file mode 100644
index 0000000..45fb7da
--- /dev/null
+++ b/src/api/fetchHfRepos.js
@@ -0,0 +1,105 @@
+import { normalizeTag, filterDisplayTags } from '../utils/normalizeTag.js';
+import { handleError } from '../ui/render.js';
+import { filterNewAdditionalEntries } from '../utils/filterNewAdditionalEntries.js';
+
+/**
+ * Function for fetching Hugging Face repositories (models, datasets, or spaces) from the specified organization.
+ * It fetches both org-owned and additional repos, processes metadata, and determines if a repo is "new" based on the provided refresh interval.
+ * @async
+ * @param {string} repoType - The type of repository to fetch ("datasets", "models", or "spaces").
+ * @param {Array} additionalHfRepos - Array of Hugging Face repos, by type to include in addition to org-owned repos
+ * @param {string} apiBaseUrl - The base URL for the Hugging Face API
+ * @param {string} hfOrgName - The Hugging Face organization name
+ * @param {number} refreshIntervalDays - The cutoff in days for determining if a repo is "new"
+ * @returns {Promise} processedItems - A promise resolving to an array of Hugging Face repositories with metadata
+ */
+export async function fetchHfRepos(
+ repoType,
+ additionalHfRepos,
+ apiBaseUrl,
+ hfOrgName,
+ refreshIntervalDays
+) {
+
+ try {
+ // hugging face api requests for datasets/models/spaces
+ const response = await fetch(`${apiBaseUrl}${repoType}?author=${hfOrgName}&full=true`);
+ if (!response.ok) {
+ throw new Error(`HTTP error! status: ${response.status}`);
+ }
+ let hfItems = await response.json();
+
+ // Fetch additional HF repos of this type from outside the org
+ const additionalForType = additionalHfRepos.filter(entry => entry.type === repoType);
+ if (additionalForType.length) {
+ const existingIds = new Set(hfItems.map(item => item.id));
+ const toFetch = filterNewAdditionalEntries(existingIds, additionalForType);
+
+ const fetched = await Promise.all(
+ toFetch.map(entry =>
+ fetch(`${apiBaseUrl}${repoType}/${entry.repo}`)
+ .then(r => {
+ if (!r.ok) {
+ console.warn(`Failed to fetch additional HF repo "${entry.repo}": HTTP ${r.status}`);
+ return null;
+ }
+ return r.json();
+ })
+ .catch(err => {
+ console.warn(`Network error fetching additional HF repo "${entry.repo}":`, err);
+ return null;
+ })
+ )
+ );
+ hfItems = [...hfItems, ...fetched.filter(item => item && !existingIds.has(item.id))];
+ }
+
+ // Step 2: If we are fetching models, get the full details for each one.
+ if (repoType === 'models') {
+ const detailPromises = hfItems.map(item =>
+ fetch(`${apiBaseUrl}models/${item.id}`).then(res => {
+ if (!res.ok) {
+ console.error(`Failed to fetch details for ${item.id}`);
+ return null; // Return null for failed requests
+ }
+ return res.json();
+ })
+ );
+
+ // Wait for all detail requests to complete in parallel.
+ const detailedItems = await Promise.all(detailPromises);
+
+ // Filter out any models that failed to fetch and assign the detailed list.
+ hfItems = detailedItems.filter(Boolean);
+ }
+ // Process the data to include metadata and a 'new' flag
+ const processedItems = hfItems.map(item => {
+ const createdAt = new Date(item.createdAt);
+ const lastModified = new Date(item.lastModified);
+ const isNew = (new Date() - createdAt) / (1000 * 60 * 60 * 24) < refreshIntervalDays;
+
+ // Extract tags from the YAML metadata (handling different structures)
+ const rawTags = (item.cardData?.tags || item.tags || []).map(t => String(t).toLowerCase());
+ const tags = [...new Set(rawTags.flatMap(t => normalizeTag(t)).filter(Boolean))];
+ const displayTags = filterDisplayTags(rawTags);
+
+ return {
+ ...item,
+ repoType,
+ createdAt,
+ lastModified,
+ isNew,
+ archived: false,
+ likes: item.likes || 0,
+ tags,
+ rawTags,
+ displayTags
+ };
+ });
+
+ return processedItems;
+ } catch (error) {
+ handleError(error, `Failed to fetch ${repoType} from Hugging Face. Please check your network connection or the API.`);
+ return [];
+ }
+}
diff --git a/src/api/fetchStats.js b/src/api/fetchStats.js
new file mode 100644
index 0000000..c43c47d
--- /dev/null
+++ b/src/api/fetchStats.js
@@ -0,0 +1,42 @@
+/**
+ * Fetches statistics for the Catalog repository itself (Stars, Forks, Version)
+ * populates the badge in the top right corner.
+ * @param {string} repoApiUrl - The base API URL for the code platform (e.g., GitHub API URL)
+ * @param {string} organizationName - The organization name for the catalog repository
+ * @param {string} catalogRepoName - The repository name for the catalog itself
+ * @param {string} releaseSuffix - The suffix used for fetching the latest release information (e.g., 'releases/latest')
+ * @returns {Promise} - A promise that resolves when the stats have been fetched and displayed
+ */
+export const fetchCatalogStats = async (repoApiUrl, organizationName, catalogRepoName, releaseSuffix) => {
+ // Helper: Updates text, shows the specific stat, and ensures the divider is visible
+ const update = (textId, containerId, value) => {
+ const el = document.getElementById(textId);
+ const container = document.getElementById(containerId);
+ if (el && container && value !== undefined) {
+ el.innerText = value;
+ if (value != 0) {
+ container.classList.remove('hidden');
+ container.classList.add('flex');
+ }
+ }
+ };
+
+ // Need to URL-encode the owner/repo for GitLab API ('%2F' instead of '/'), but need '/' for GitHub and Codeberg.
+ const ownerRepo = repoApiUrl.includes("gitlab") ? `${organizationName}%2F${catalogRepoName}` : `${organizationName}/${catalogRepoName}`;
+
+ try {
+ // platform isn't passed, forks_count is shared
+ // 1. Get Stars & Forks
+ const repo = await fetch(`${repoApiUrl}${ownerRepo}`).then(r => r.ok ? r.json() : {});
+ const star_count = repo.stargazers_count ?? repo.stars_count ?? repo.star_count;
+ if (star_count !== undefined) update('gh-stars', 'gh-star-container', star_count);
+ if (repo.forks_count !== undefined) update('gh-forks', 'gh-fork-container', repo.forks_count);
+
+ // 2. Get Version (Tag)
+ const release = await fetch(`${repoApiUrl}${ownerRepo}/${releaseSuffix}`).then(r => r.ok ? r.json() : {});
+ if (release.tag_name !== undefined) update('gh-tag', 'gh-version-container', release.tag_name);
+
+ } catch (e) {
+ console.warn("Could not fetch Code Repo stats", e);
+ }
+};
diff --git a/src/assets/GitHub_Invertocat.svg b/src/assets/GitHub_Invertocat.svg
new file mode 100755
index 0000000..5300969
--- /dev/null
+++ b/src/assets/GitHub_Invertocat.svg
@@ -0,0 +1,10 @@
+
+
+
+
+
+
+
+
diff --git a/src/assets/codeberg-logo_icon_white.svg b/src/assets/codeberg-logo_icon_white.svg
new file mode 100644
index 0000000..74a0fa9
--- /dev/null
+++ b/src/assets/codeberg-logo_icon_white.svg
@@ -0,0 +1,164 @@
+
+
+ Codeberg logo
+
+
+
+
+
+
+
+
+
+
+
+ image/svg+xml
+ Codeberg logo
+
+
+ Robert Martinez
+
+
+
+
+ Codeberg and the Codeberg Logo are trademarks of Codeberg e.V.
+
+
+ 2020-04-09
+
+
+ Codeberg e.V.
+
+
+ codeberg.org
+
+
+
+
+
+
+
+
diff --git a/src/assets/gitlab-logo-700-rgb.svg b/src/assets/gitlab-logo-700-rgb.svg
new file mode 100644
index 0000000..72e3232
--- /dev/null
+++ b/src/assets/gitlab-logo-700-rgb.svg
@@ -0,0 +1 @@
+ tags inserted after underscores.
+ */
+const addWordBreakOpportunities = (str) => {
+ if (!str) return "";
+ // Replace underscores with underscore + word break opportunity
+ return str.replace(/_/g, '_');
+};
+
+/**
+ * Renders a single item card (code, dataset, model, or space) to HTML.
+ * @param {Object} item - The item object to render.
+ * @returns {string} The HTML string for the item card.
+ */
+const renderHubItemCard = (item) => {
+ const lastUpdatedDate = new Date(item.lastModified).toLocaleDateString();
+ const tagsHtml = (item.displayTags || item.rawTags || []).map(tag =>
+ `${escapeHTML(tag)} `
+ ).join('');
+
+ // Use pretty_name for the heading, with a fallback
+ // HF API keys for CardData: https://huggingface.co/docs/huggingface_hub/main/en/package_reference/cards#huggingface_hub.CardData
+ // datasets have pretty_name, models have model_name, spaces have title
+ const prettyName = item.cardData?.pretty_name || item.cardData?.model_name || item.cardData?.title || item.id.split('/')[1];
+
+ // Use the description from cardData, with fallbacks
+ const displayDescription = item.cardData?.description || item.cardData?.model_description || item.description || 'No description provided.';
+
+ // Construct the correct URL based on the repository type
+ let itemUrl;
+
+ switch (item.repoType) {
+ case "code":
+ itemUrl = item.html_url;
+ break;
+ case "datasets":
+ itemUrl = `https://huggingface.co/datasets/${item.id}`;
+ break;
+ case "spaces":
+ itemUrl = `https://huggingface.co/spaces/${item.id}`;
+ break;
+ case "models":
+ itemUrl = `https://huggingface.co/${item.id}`;
+ break;
+ default:
+ // fallback for "all"
+ itemUrl = `https://huggingface.co/${item.id}`;
+ break;
+ }
+
+ // stars for code repos
+ const badgeHtml = (() => {
+ if (item.isNew) {
+ return `
+ New!
+ `;
+ }
+
+ if (typeof item.cardData?.stars === "number" && item.cardData.stars > 0) {
+ return `
+ ā ${item.cardData.stars}
+ `;
+ }
+
+ if (typeof item.likes === "number" && item.likes > 0) {
+ return `
+
+ ā¤ļø ${item.likes}
+ `;
+ }
+ return "";
+ })();
+
+ const escapedTitle = escapeHTML(prettyName);
+ const displayTitle = addWordBreakOpportunities(escapedTitle);
+
+ return `
+
+
+
+
+ ${escapeHTML(displayDescription)}
+
+
+
+
+ ${tagsHtml}
+
+
+ Updated: ${lastUpdatedDate}
+ ${item.archived ? `Archived ` : ''}
+
+
+
+
Error loading items.
+
${escapeHTML(message)}
+
Spaces Stars PyTorch All