Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 0 additions & 29 deletions src/lib/datasetHistory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,6 @@ export interface DatasetHistoryAnnotation extends GrowthAnnotation {
metric: 'datasetCount' | 'imageCount';
}

function formatShortDate(iso: string): string {
const d = new Date(`${iso}T00:00:00Z`);
return d.toLocaleDateString('en-US', { month: 'short', day: 'numeric', timeZone: 'UTC' });
}

// Buckets points into fixed-length windows (in days) counted from the earliest point's date,
// keeping the most recent point in each window. Snapshots are appended by hand at whatever
// cadence someone happens to run the script, so this is what turns that irregular series into an
Expand Down Expand Up @@ -57,27 +52,3 @@ export function remapAnnotations<T extends { date: string }>(
return { ...annotation, atPeriod: closest.period };
});
}

// Builds one point per calendar day for the last `days` days (ending today), forward-filling
// each day with the most recent snapshot on or before it — snapshots aren't taken daily, so
// without this a daily chart would just be gaps.
export function lastNDaysForwardFilled(points: DatasetHistoryPoint[], days: number): DatasetHistoryPoint[] {
if (points.length === 0) return [];
const sorted = [...points].sort((a, b) => a.date.localeCompare(b.date));
const today = new Date();
today.setUTCHours(0, 0, 0, 0);

const result: DatasetHistoryPoint[] = [];
let cursor = 0;
let carried = sorted[0];
for (let i = days - 1; i >= 0; i--) {
const day = new Date(today.getTime() - i * 86_400_000);
const dayIso = day.toISOString().slice(0, 10);
while (cursor < sorted.length && sorted[cursor].date <= dayIso) {
carried = sorted[cursor];
cursor += 1;
}
result.push({ ...carried, period: formatShortDate(dayIso), date: dayIso });
}
return result;
}
11 changes: 11 additions & 0 deletions src/lib/datasets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -730,6 +730,7 @@ export interface DatasetStats {
datasetCount: number;
imageCount: number;
taskTypeCount: number;
cropCount: number;
}

// `parent_dataset` means two different things depending on the record. On vision datasets it
Expand Down Expand Up @@ -793,11 +794,21 @@ export function computeDatasetStats(datasets: Dataset[]): DatasetStats {
.map((dataset) => dataset.machine_learning_task)
.filter((task): task is string => Boolean(task)),
).size;
// Distinct crops named by top-level datasets. Species rolled up from child datasets
// (`child_crop_types`) are left out: iNatAg alone contributes ~2,900 wild plant species,
// which would swamp the count and aren't crops.
const cropCount = new Set(
topLevel
.flatMap((dataset) => dataset.crop_types ?? [])
.map((crop) => crop.trim().toLowerCase())
.filter(Boolean),
).size;

return {
datasetCount: topLevel.length,
imageCount,
taskTypeCount,
cropCount,
};
}

Expand Down
121 changes: 42 additions & 79 deletions src/pages/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import GrowthLineChart from '../components/GrowthLineChart';
import DistributionBarChart from '../components/DistributionBarChart';
import {
bucketByInterval,
lastNDaysForwardFilled,
remapAnnotations,
type DatasetHistoryAnnotation,
type DatasetHistoryPoint,
Expand Down Expand Up @@ -174,19 +173,18 @@ function StatsRow() {
<span className={styles.statLabel}>Labeled images</span>
</div>
<div className={styles.statBlock}>
<span className={styles.statValue}>{stats.taskTypeCount.toLocaleString()}</span>
<span className={styles.statLabel}>CV task types</span>
<span className={styles.statValue}>{stats.cropCount.toLocaleString()}</span>
<span className={styles.statLabel}>Crops covered</span>
</div>
</section>
);
}

// Every point in dataset_history.json is a hand-run snapshot at whatever cadence someone
// happened to run the script, so the long-range charts are bucketed down to one point every two
// weeks for an even x-axis, and annotations are re-anchored to whichever bucketed point survives
// closest to their original snapshot.
// happened to run the script, so the charts are bucketed down to one point every two weeks for
// an even x-axis, and annotations are re-anchored to whichever bucketed point survives closest
// to their original snapshot.
const BIWEEKLY_DAYS = 14;
const RECENT_DAYS = 7;

function GrowthCharts() {
const history = useDatasetHistory();
Expand All @@ -195,82 +193,47 @@ function GrowthCharts() {
() => (history ? bucketByInterval(history.points, BIWEEKLY_DAYS) : []),
[history],
);
const recent = useMemo(
() => (history ? lastNDaysForwardFilled(history.points, RECENT_DAYS) : []),
[history],
);

return (
<>
<section className={styles.chartsSection}>
<p className={styles.sectionLabel}>Growth over time</p>
<div className={styles.chartsGrid}>
<div className={styles.chartPanel}>
<h3 className={styles.chartPanelTitle}>Top-level datasets over time</h3>
{history ? (
<GrowthLineChart
points={biweekly.map((p) => ({ period: p.period, date: p.date, value: p.datasetCount }))}
annotations={remapAnnotations(
history.annotations.filter((a) => a.metric === 'datasetCount'),
history.points,
biweekly,
)}
color="primary"
yaxisTitle="Top-level datasets"
/>
) : (
<div className={styles.chartLoading}>Loading…</div>
)}
</div>
<div className={styles.chartPanel}>
<h3 className={styles.chartPanelTitle}>Images over time</h3>
{history ? (
<GrowthLineChart
points={biweekly.map((p) => ({ period: p.period, date: p.date, value: p.imageCount }))}
annotations={remapAnnotations(
history.annotations.filter((a) => a.metric === 'imageCount'),
history.points,
biweekly,
)}
color="teal"
yaxisTitle="Images"
/>
) : (
<div className={styles.chartLoading}>Loading…</div>
)}
</div>
<section className={styles.chartsSection}>
<p className={styles.sectionLabel}>Growth over time</p>
<div className={styles.chartsGrid}>
<div className={styles.chartPanel}>
<h3 className={styles.chartPanelTitle}>Top-level datasets over time</h3>
{history ? (
<GrowthLineChart
points={biweekly.map((p) => ({ period: p.period, date: p.date, value: p.datasetCount }))}
annotations={remapAnnotations(
history.annotations.filter((a) => a.metric === 'datasetCount'),
history.points,
biweekly,
)}
color="primary"
yaxisTitle="Top-level datasets"
/>
) : (
<div className={styles.chartLoading}>Loading…</div>
)}
</div>
</section>
<section className={styles.chartsSection}>
<p className={styles.sectionLabel}>Last 7 days</p>
<div className={styles.chartsGrid}>
<div className={styles.chartPanel}>
<h3 className={styles.chartPanelTitle}>Top-level datasets</h3>
{history ? (
<GrowthLineChart
points={recent.map((p) => ({ period: p.period, date: p.date, value: p.datasetCount }))}
color="primary"
yaxisTitle="Top-level datasets"
/>
) : (
<div className={styles.chartLoading}>Loading…</div>
)}
</div>
<div className={styles.chartPanel}>
<h3 className={styles.chartPanelTitle}>Images</h3>
{history ? (
<GrowthLineChart
points={recent.map((p) => ({ period: p.period, date: p.date, value: p.imageCount }))}
color="teal"
yaxisTitle="Images"
/>
) : (
<div className={styles.chartLoading}>Loading…</div>
)}
</div>
<div className={styles.chartPanel}>
<h3 className={styles.chartPanelTitle}>Images over time</h3>
{history ? (
<GrowthLineChart
points={biweekly.map((p) => ({ period: p.period, date: p.date, value: p.imageCount }))}
annotations={remapAnnotations(
history.annotations.filter((a) => a.metric === 'imageCount'),
history.points,
biweekly,
)}
color="teal"
yaxisTitle="Images"
/>
) : (
<div className={styles.chartLoading}>Loading…</div>
)}
</div>
</section>
</>
</div>
</section>
);
}

Expand Down
Loading