diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3558fe4..7152e0b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,7 +37,12 @@ jobs: path: Lean - name: Move Lean - run: mv Lean ../Lean + # The runners are self-hosted and ../Lean sits outside the checkout, so it survives + # between runs. Left in place, mv moves into it instead of renaming, and the step + # fails on whichever runner has already built this repo once. + run: | + rm -rf ../Lean + mv Lean ../Lean - name: Run build and tests run: | diff --git a/DataProcessing/DataProcessing.csproj b/DataProcessing/DataProcessing.csproj index 697ba6a..92ba250 100644 --- a/DataProcessing/DataProcessing.csproj +++ b/DataProcessing/DataProcessing.csproj @@ -16,4 +16,9 @@ + + diff --git a/DataProcessing/Program.cs b/DataProcessing/Program.cs index 6367eff..ccc4934 100644 --- a/DataProcessing/Program.cs +++ b/DataProcessing/Program.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -14,7 +14,9 @@ */ using QuantConnect.Configuration; +using QuantConnect.DataSource; using QuantConnect.Logging; +using QuantConnect.Util; using System; using System.Diagnostics; using System.Globalization; @@ -23,11 +25,60 @@ namespace QuantConnect.DataProcessing { /// - /// Console program to convert from raw SEC data to a formatted form usable by LEAN + /// Console program to convert from raw SEC data to a formatted form usable by LEAN. + /// + /// The repository ships two unrelated SEC datasets and the "dataset-name" config key selects + /// which one a run processes: + /// + /// - "reports" (the default): the 10-K, 10-Q and 8-K filings, downloaded with + /// and converted with . + /// - "13f" (): Form 13F institutional holdings, aggregated per + /// security from the quarterly structured data sets. + /// + /// The default keeps a job that sets no dataset-name doing exactly what it did before the key + /// existed. /// public class Program { - public static void Main() + /// + /// The "dataset-name" config value that selects the shipped SEC reports dataset, and the + /// value assumed when the key is not set. + /// + private const string ReportsDatasetName = "reports"; + + /// Config key that asks the 13F run to rebuild the whole history instead of one date. + internal const string RebuildHistoryKey = "sec-13f-rebuild-history"; + + /// + /// Entrypoint of the program. The exit code is returned rather than handed to + /// from inside the work: that call does not unwind the stack, + /// so every finally block written for the failure paths would be skipped on all of them. + /// + /// Zero on success, one on any failure + public static int Main() + { + var dataset = Config.Get("dataset-name", ReportsDatasetName).Trim().ToLowerInvariant(); + + switch (dataset) + { + case ReportsDatasetName: + return ProcessReports(); + + case SEC13FDownloader.DatasetName: + return Process13F(); + + default: + Log.Error($"DataProcessing.Main(): Unknown dataset-name '{dataset}'. Valid options: " + + $"{ReportsDatasetName}, {SEC13FDownloader.DatasetName}"); + return 1; + } + } + + /// + /// Downloads and converts the SEC reports dataset for the deployment date. + /// + /// Zero on success, one on any failure + private static int ProcessReports() { var processingDateValue = Environment.GetEnvironmentVariable("QC_DATAFLEET_DEPLOYMENT_DATE"); var processingDate = DateTime.ParseExact(processingDateValue, "yyyyMMdd", CultureInfo.InvariantCulture); @@ -62,10 +113,115 @@ public static void Main() catch (Exception e) { Log.Error(e, $"DataProcessing.Main(): {processingDate} Exception while processing SEC data"); - Environment.Exit(1); + return 1; + } + + return 0; + } + + /// + /// Downloads and converts the Form 13F institutional holdings dataset for the deployment date, + /// folding it into the published history, or rebuilds the whole history when asked to. + /// + /// Zero on success, one on any failure + private static int Process13F() + { + // Output root: {temp-output-directory}/alternative/sec. The downloader writes its report + // under the folder its data class names. + var destinationDirectory = Path.Combine( + Config.Get("temp-output-directory", "/temp-output-directory"), + "alternative", + SEC13FDownloader.VendorName); + + // The published history an incremental run merges its rows into. The destination is + // handed to the job empty, so reading history back from there would find nothing. + var processedDataDirectory = Path.Combine( + Config.Get("processed-data-directory", Globals.DataFolder), + "alternative", + SEC13FDownloader.VendorName); + + // Where downloads land. The job archives this folder after every run, as it does the + // reports dataset's feed archives, but does not restore it before the next. + var rawDataDirectory = Path.Combine( + Config.Get("raw-data-folder", "/raw"), + "alternative", + SEC13FDownloader.VendorName); + + if (!TryParseDeploymentDate(out var deploymentDate)) + { + return 1; + } + + Log.Trace($"DataProcessing.Process13F(): writing {SEC13FDownloader.DatasetName} to {destinationDirectory}" + + (deploymentDate == null ? " for the full history" : $" for {deploymentDate:yyyy-MM-dd}")); + + var timer = Stopwatch.StartNew(); + SEC13FDownloader downloader; + try + { + downloader = new SEC13FDownloader(destinationDirectory, processedDataDirectory, deploymentDate, + rawDataDirectory); + } + catch (Exception err) + { + Log.Error(err, $"DataProcessing.Process13F(): The {SEC13FDownloader.DatasetName} downloader failed to be constructed"); + return 1; + } + + try + { + downloader.Run(); + + timer.Stop(); + Log.Trace($"DataProcessing.Process13F(): Conversion finished in time {timer.Elapsed}"); + return 0; + } + catch (Exception err) + { + Log.Error(err, $"DataProcessing.Process13F(): The {SEC13FDownloader.DatasetName} downloader exited unexpectedly"); + return 1; + } + finally + { + downloader.DisposeSafely(); + } + } + + /// + /// Reads the deployment date the job runs for. A missing date is a misconfigured job rather + /// than a request for the whole history, which is asked for by name with + /// . + /// + /// The date, or null when the run rebuilds the whole history + /// True when the date is well formed, or absent with the rebuild asked for + internal static bool TryParseDeploymentDate(out DateTime? deploymentDate) + { + deploymentDate = null; + + var raw = Environment.GetEnvironmentVariable("QC_DATAFLEET_DEPLOYMENT_DATE"); + if (string.IsNullOrWhiteSpace(raw)) + { + if (Config.GetBool(RebuildHistoryKey)) + { + return true; + } + + Log.Error("DataProcessing.TryParseDeploymentDate(): QC_DATAFLEET_DEPLOYMENT_DATE is not set. Set it, or " + + $"set \"{RebuildHistoryKey}\": true to rebuild the whole history"); + return false; + } + + // A malformed date must not quietly become a full history run: that would turn a daily + // job into a complete refetch of five gigabytes without anyone noticing. + if (!DateTime.TryParseExact(raw.Trim(), "yyyyMMdd", CultureInfo.InvariantCulture, + DateTimeStyles.None, out var parsed)) + { + Log.Error($"DataProcessing.TryParseDeploymentDate(): QC_DATAFLEET_DEPLOYMENT_DATE '{raw}' is not yyyyMMdd"); + return false; } - Environment.Exit(0); + deploymentDate = parsed; + return true; } } } diff --git a/DataProcessing/SEC13FClosePrices.cs b/DataProcessing/SEC13FClosePrices.cs new file mode 100644 index 0000000..b2c09d3 --- /dev/null +++ b/DataProcessing/SEC13FClosePrices.cs @@ -0,0 +1,94 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; + +namespace QuantConnect.DataProcessing +{ + /// + /// Raw closing prices from LEAN's coarse universe files, one per trading day, which is what a + /// reported VALUE over SSHPRNAMT is checked against. The close of the quarter's last trading day + /// is known before any filing for that quarter can be made, so the check needs no other filing + /// and gives the same answer on the day a filing is read as in a rebuild years later. + /// + internal sealed class SEC13FClosePrices + { + /// Days walked back from a quarter end for its last trading day: a long weekend and a holiday. + private const int DaysToLookBack = 7; + + private readonly string _coarseDirectory; + private readonly Dictionary> _closesByDay = new(); + + public SEC13FClosePrices(string coarseDirectory) + { + _coarseDirectory = coarseDirectory; + } + + /// Whether the coarse files are there at all, so the caller can say so once. + public bool Available => Directory.Exists(_coarseDirectory); + + /// + /// The raw close of a security on the last trading day on or before the quarter end, or + /// null. Never a day after the filing date: a period typed years ahead would otherwise read + /// a price nobody had when the filing was made. + /// + public decimal? Close(SecurityIdentifier security, DateTime periodEnd, DateTime filingDate) + { + var last = periodEnd.Date < filingDate.Date ? periodEnd.Date : filingDate.Date; + for (var back = 0; back < DaysToLookBack; back++) + { + var closes = ClosesOn(last.AddDays(-back)); + if (closes != null) + { + // The last trading day decides: a security missing from it did not trade then. + return closes.TryGetValue(security.ToString(), out var close) && close > 0m ? close : null; + } + } + + return null; + } + + /// The closes of one trading day, keyed by security identifier, or null when the day has no file. + private Dictionary ClosesOn(DateTime day) + { + if (_closesByDay.TryGetValue(day, out var closes)) + { + return closes; + } + + var path = Path.Combine(_coarseDirectory, $"{day.ToString(DateFormat.EightCharacter, CultureInfo.InvariantCulture)}.csv"); + if (File.Exists(path)) + { + closes = new Dictionary(StringComparer.Ordinal); + + // sid, ticker, close, volume, dollar volume, has fundamentals, price factor, split factor + foreach (var line in File.ReadLines(path)) + { + var fields = line.Split(','); + if (fields.Length > 2 && decimal.TryParse(fields[2], NumberStyles.Any, CultureInfo.InvariantCulture, out var close)) + { + closes[fields[0]] = close; + } + } + } + + _closesByDay[day] = closes; + return closes; + } + } +} diff --git a/DataProcessing/SEC13FDownloader.cs b/DataProcessing/SEC13FDownloader.cs new file mode 100644 index 0000000..0048539 --- /dev/null +++ b/DataProcessing/SEC13FDownloader.cs @@ -0,0 +1,2887 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using QuantConnect.Configuration; +using QuantConnect.Data.Auxiliary; +using QuantConnect.DataSource; +using QuantConnect.Interfaces; +using QuantConnect.Lean.Engine.DataFeeds; +using QuantConnect.Logging; +using QuantConnect.Securities; +using QuantConnect.Util; + +[assembly: InternalsVisibleTo("Tests")] + +namespace QuantConnect.DataProcessing +{ + /// + /// Converts Form 13F filings into LEAN's per-security files and one universe file per release + /// date. Without QC_DATAFLEET_DEPLOYMENT_DATE it rebuilds the whole history, from the SEC's + /// structured data sets through the last window published and from EDGAR's daily indexes after + /// it; with it, it reads that day from EDGAR, with any recent day whose index came late, and + /// folds them into the published history. + /// + /// Besides the data folders it reads these config keys: sec-user-agent-company-name and + /// sec-user-agent-company-email, which the SEC asks automated readers for; sec-13f-rebuild-history, + /// which a run without a deployment date needs; and, for checks only, sec-13f-edgar-from, + /// sec-13f-edgar-until and sec-13f-rebuild-from, which move the rebuild's EDGAR days and skip the + /// older data sets. From the data folder it reads the map files and security-database.csv, which + /// resolve CUSIPs and tickers, and the coarse universe files, whose quarter-end closes decide + /// VALUE's unit and vet the crosswalk. + /// + public class SEC13FDownloader : IDisposable + { + /// + /// The "dataset-name" config value that selects this downloader. Spelled the same as the + /// data class's ReportFolder, which stays the single source of truth for the output path. + /// + public const string DatasetName = "13f"; + + /// Vendor name (matches the alternative/<vendor> data path). + public static string VendorName => "sec"; + + private const string DataSetsPageUrl = "https://www.sec.gov/data-research/sec-markets-data/form-13f-data-sets"; + private const string SecBaseUrl = "https://www.sec.gov"; + + + private const int MaxRetries = 8; + + // A 13F-NT is a notice that the manager's holdings are reported on someone else's filing. It + // carries no INFOTABLE at all, so it is not a zero position and must not become one. + private const string NoticeSubmissionTypePrefix = "13F-NT"; + + // A 13F-HR/A amends the manager's report for the quarter, and most restate all of it, so an + // amendment's lines count only for a security the manager had not reported yet. A + // restatement's corrections to a position already counted are not applied; see AdmitAmendment. + private const string AmendmentSubmissionTypeSuffix = "/A"; + + // EDGAR lists a day's filings in its daily index at about 22:05 ET (02:02 to 02:07 UTC the + // next morning over six business days measured in September 2026). The daily job reads that + // index at 01:00 ET the next day with a one hour timeout (schedule "0 1 * * 2-6", Date + // Offset 1), so a filing is published the day after its FILING_DATE at ReleaseTimeOfDay. + // Stamping it any earlier would let a backtest read it before the live job had it. + internal static readonly TimeSpan AvailableAfterFilingDate = TimeSpan.FromDays(1) + SEC13F.ReleaseTimeOfDay; + + /// Config key that starts the rebuild's EDGAR days on this yyyyMMdd instead of after the last data set. + internal const string EdgarFromKey = "sec-13f-edgar-from"; + + /// Config key that ends the rebuild's EDGAR days on this yyyyMMdd instead of yesterday. + internal const string EdgarUntilKey = "sec-13f-edgar-until"; + + /// + /// Config key that makes the rebuild skip the data sets ending before this yyyyMMdd. For checks + /// only: the rules do not depend on the old years, and cutting them takes a rebuild from ten + /// minutes to one or two. A published history always starts in 2013. + /// + internal const string RebuildFromKey = "sec-13f-rebuild-from"; + + /// The EDGAR days already folded into the published history, one yyyyMMdd per line. + private const string EdgarStateFileName = "edgar-days.txt"; + + /// How far back a daily run looks for a day whose index EDGAR published late. + private const int EdgarLookbackDays = 10; + + private const string ReleaseFormat = "yyyyMMdd HH:mm"; + private const string PeriodFormat = "yyyyMMdd"; + + // A staged row: release, period, the eight measures and the confidential flag. + private const int FirstValueColumn = 2; + private const int ValueColumnCount = 8; + private const int PublishedColumnCount = FirstValueColumn + ValueColumnCount + 1; + + // A published line: the release stamp, how many quarters it restates, and then that many + // groups of columns, one per quarter. Every quarter a release restates travels in one line, + // since LEAN hands an algorithm a single data point per security per timestamp. + private const int PointHeaderColumns = 2; + private const int GroupColumnCount = ValueColumnCount + 2; + + // N-PORT quarters folded into the crosswalk: a year reaches every security still trading. + private const int NPortQuartersToFold = 4; + + /// Days after a quarter end that managers have to file their 13F for it. + private const int FilingDeadlineDays = 45; + + // The SEC rule: VALUE in thousands for filings before this date, whole dollars from it. + // DetectValueUnits checks each filing against it. + private static readonly DateTime ValueInWholeDollarsFrom = new(2023, 1, 1); + + // How far, in log10, a crosswalk group's median price may sit from the close times a power + // of a thousand and still be that security: about three times either way. Farther is another + // security, as DeFi Technologies at $2 reaching the $129 Hashdex DEFI ETF through its ticker. + private const double PriceMatchTolerance = 0.5; + + private static readonly string[] SecDateFormats = { "dd-MMM-yyyy", "d-MMM-yyyy", "yyyy-MM-dd", "MM/dd/yyyy" }; + + private static readonly Regex ArchiveLinkRegex = new( + @"href=""(?[^""]*?/(?[^""/]+_form13f\.zip))""", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + + private static readonly Regex QuarterArchiveRegex = new( + @"^(?\d{4})q(?[1-4])_form13f\.zip$", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + + private static readonly Regex WindowArchiveRegex = new( + @"^(?\d{2}[a-z]{3}\d{4})-(?\d{2}[a-z]{3}\d{4})_form13f\.zip$", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + + private readonly string _destinationDirectory; + private readonly string _processedDataDirectory; + private readonly string _archiveCacheDirectory; + private readonly DateTime? _deploymentDate; + + private readonly HttpClient _client; + private bool _userAgentSet; + + // The SEC rate limits automated readers to ten requests a second. + private readonly RateGate _rateGate = new(10, TimeSpan.FromSeconds(1)); + + private readonly IMapFileProvider _mapFileProvider; + + // The security database's rows by CUSIP body and by ISIN, every row kept, since the database + // repeats identifiers across the listings one company has had. See TradingDefinition. + private readonly Dictionary> _definitionsByCusip; + private readonly Dictionary> _definitionsByIsin; + + /// + /// CUSIP and filing date to security, misses included, since each miss scans the whole + /// security database. The date is part of the key because resolution is point in time; the + /// cache is cleared per archive so it stays the size of one window. + /// + private readonly Dictionary<(string Cusip, DateTime Date), SecurityIdentifier> _resolvedCusips = new(); + + /// CUSIPs already tallied in the resolution summary, so its totals stay per CUSIP. + private readonly HashSet _countedCusips = new(StringComparer.Ordinal); + + /// Map file per security, so the point-in-time ticker costs one lookup per date. + private readonly Dictionary _mapFiles = new(); + + /// Accessions already folded in, so a submission can never be counted twice. + private readonly HashSet _processedAccessions = new(StringComparer.Ordinal); + + /// + /// Increments waiting to be staged, flushed per archive. Keyed by security and then by + /// (period, release), because several CUSIPs can reach one security and their contributions + /// are summed here. Each keeps the ticker of its day, which is the file it lands in. + /// + private readonly Dictionary> + _pendingSecurityRows = new(StringComparer.Ordinal); + + /// + /// Where increments wait, one file per security, until the finalize pass. Per security + /// because the running total belongs to the security and not to a ticker, and outside the + /// output folder because everything there is published. + /// + private readonly string _stagingDirectory = + Path.Combine(Path.GetTempPath(), "sec-13f-staging", Guid.NewGuid().ToString("N")); + + /// Every security with increments staged, which is every security this run touched. + private readonly HashSet _stagedSecurities = new(StringComparer.Ordinal); + + /// + /// Amendments admitted in this archive, as (filer key, filing date), so one amendment naming + /// several CUSIPs of a security is admitted for all of them. + /// + private readonly HashSet<(long FilerKey, int Stamp)> _admittedAmendments = new(); + + /// Ticker and date to the security the map files say owns that ticker then, per archive. + private readonly Dictionary<(string Ticker, DateTime Date), string> _tickerOwners = new(); + + private long _conflictingTickers; + private decimal _conflictingValue; + private readonly List _conflictSample = new(); + private long _unchangedGroups; + + /// + /// CUSIP to the ticker the SEC's N-PORT filings report for it. Built on first use rather than + /// at construction: the archives may all resolve through the security database, and building + /// it downloads a few hundred megabytes per quarter folded in. Settable for tests. + /// + internal Dictionary TickerCrosswalk + { + // Read from the published copy and written with the output, since the raw folder is not + // restored between runs and the next run would otherwise rebuild it from 1.8 GB of N-PORT. + get => _tickerCrosswalk ??= SEC13FTickerCrosswalk.Load(_processedDataDirectory, _destinationDirectory, + NPortQuartersToFold, UrlExists, (url, name) => DownloadFile(url, name, _archiveCacheDirectory)); + set => _tickerCrosswalk = value; + } + + private Dictionary _tickerCrosswalk; + + /// CUSIP to the security its crosswalk ticker named when the funds reported it. + private readonly Dictionary _crosswalkSecurities = new(StringComparer.Ordinal); + + private long _resolvedByCusip; + private long _resolvedByIsin; + private long _resolvedByTicker; + private long _unresolvedGroups; + private long _malformedCusips; + private decimal _resolvedValue; + private decimal _unresolvedValue; + private readonly HashSet _unresolvedCusips = new(StringComparer.Ordinal); + + /// + /// CUSIPs and filing dates resolved through the crosswalk rather than the security database, + /// cleared with the resolution cache. Their groups are checked against the close, since a fund + /// administrator's ticker can name another company. + /// + private readonly HashSet<(string Cusip, DateTime Date)> _crosswalkResolutions = new(); + + private long _debtCusipsRejected; + private long _mismatchedGroups; + private decimal _mismatchedValue; + + /// The quarter-end closes that decide VALUE's unit and vet crosswalk resolutions. + private readonly SEC13FClosePrices _closePrices; + + /// EDGAR days already folded into the history, published so a daily run never reads one twice. + private readonly SortedSet _edgarDays = new(); + + /// EDGAR's daily index directory listings, by folder URL, read once a run. + private readonly Dictionary> _edgarListings = new(StringComparer.Ordinal); + + private readonly DateTime? _edgarFrom; + private readonly DateTime? _edgarUntil; + + /// + /// Creates a new instance writing to , merging with any + /// previously processed data found in . A null + /// rebuilds the history from the data sets and EDGAR; a date + /// reads that day's filings from EDGAR, with any recent day it has not read yet. Downloads are + /// kept under so the next run finds them. + /// + public SEC13FDownloader(string destinationDirectory, string processedDataDirectory, DateTime? deploymentDate, + string rawDataDirectory = null) + { + // The folder comes from the data type rather than a literal, so the writer and the + // reader cannot drift apart. + _destinationDirectory = Path.Combine(destinationDirectory, SEC13F.ReportFolder); + _processedDataDirectory = Path.Combine(processedDataDirectory, SEC13F.ReportFolder); + _deploymentDate = deploymentDate; + + // Downloads land in the raw folder, which the job archives after every run but does not + // restore before the next, so it only saves work within a run. + _archiveCacheDirectory = rawDataDirectory == null + ? Path.Combine(Path.GetTempPath(), "sec-13f-archives") + : Path.Combine(rawDataDirectory, SEC13F.ReportFolder, "archives"); + Directory.CreateDirectory(_archiveCacheDirectory); + + _client = new HttpClient { Timeout = TimeSpan.FromMinutes(30) }; + + // Zip, not disk: the security master ships map_files_.zip, and the disk provider + // only sees loose csv files, so it would resolve almost nothing without failing. + _mapFileProvider = new LocalZipMapFileProvider(); + _mapFileProvider.Initialize(new DefaultDataProvider()); + + // The security database is in place wherever the job runs, as the map files are. Its rows are + // read here rather than through LEAN's resolver, which keeps only the first row of an + // identifier the database repeats; a data folder without it, as in the unit tests, reads as empty. + var securityDatabasePath = Path.Combine( + Globals.GetDataFolderPath("symbol-properties"), "security-database.csv"); + SecurityDefinition.TryRead(new DefaultDataProvider(), securityDatabasePath, out var definitions); + definitions ??= new List(); + + _definitionsByCusip = IndexDefinitions(definitions, definition => DatabaseCusip(definition.CUSIP)); + _definitionsByIsin = IndexDefinitions(definitions, definition => definition.ISIN); + + _closePrices = new SEC13FClosePrices(Path.Combine(Globals.DataFolder, "equity", "usa", "fundamental", "coarse")); + if (!_closePrices.Available) + { + Log.Error("SEC13FDownloader(): the coarse universe files are missing from the data folder. Without the " + + "quarter-end closes VALUE takes the SEC unit rule of each filing's date, and a crosswalk match whose " + + "CUSIP carries letters in its issue number, which only a price can confirm, is dropped."); + } + + _edgarFrom = ParseOptionalDate(Config.Get(EdgarFromKey), EdgarFromKey); + _edgarUntil = ParseOptionalDate(Config.Get(EdgarUntilKey), EdgarUntilKey); + } + + /// A yyyyMMdd config value, or null when the key is not set. + private static DateTime? ParseOptionalDate(string value, string key) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (DateTime.TryParseExact(value.Trim(), DateFormat.EightCharacter, CultureInfo.InvariantCulture, + DateTimeStyles.None, out var date)) + { + return date; + } + + throw new ArgumentException($"SEC13FDownloader(): {key} '{value}' is not yyyyMMdd"); + } + + /// + /// Runs the download/convert. Failures throw rather than return false, so the caller logs + /// the actual reason: a guard stopping the run and a network outage are not the same thing. + /// + public void Run() + { + RequireEmptyDestination(); + RequirePublishedHistoryForIncrementalRun(); + ReadFilerState(); + ReadEdgarState(); + + if (_deploymentDate == null) + { + // The data sets as far as they reach, then EDGAR day by day: the filings the daily + // job reads, with the stamps it gives them. + var archives = GetArchives(); + var edgarFrom = _edgarFrom ?? archives[archives.Count - 1].End.AddDays(1); + var rebuildFrom = ParseOptionalDate(Config.Get(RebuildFromKey), RebuildFromKey); + var selected = ArchivesBefore(archives, edgarFrom) + .Where(archive => rebuildFrom == null || archive.End >= rebuildFrom) + .ToList(); + _edgarFirstDay = edgarFrom; + Log.Trace($"SEC13FDownloader.Run(): {archives.Count} archives published, processing {selected.Count} " + + $"through {selected.LastOrDefault()?.End:yyyy-MM-dd}, then EDGAR from {edgarFrom:yyyy-MM-dd}"); + + foreach (var archive in selected) + { + ProcessArchive(archive); + FlushPendingRows(); + } + + ProcessEdgarDays(EdgarDaysToRead(edgarFrom, _edgarUntil ?? YesterdayInNewYork())); + } + else + { + // The deployment date, and any recent day whose index EDGAR published late. + ProcessEdgarDays(EdgarDaysToRead(_deploymentDate.Value.AddDays(-EdgarLookbackDays), _deploymentDate.Value)); + } + + FinalizeSecurityFiles(); + BuildUniverseFiles(); + WriteFilerState(); + WriteEdgarState(); + LogResolutionSummary(); + } + + /// + /// Reads each day's filings from EDGAR and folds them in like an archive. A weekday without an + /// index is a holiday or a late index; it is not recorded, so a later run can still read it. + /// + private void ProcessEdgarDays(List days) + { + var read = 0; + foreach (var day in days) + { + var path = SEC13FEdgarDay.Build(day, _archiveCacheDirectory, ListEdgarDirectory, GetWithRetry); + if (path == null) + { + Log.Trace($"SEC13FDownloader.ProcessEdgarDays(): EDGAR lists no index for {day:yyyy-MM-dd}"); + continue; + } + + ProcessArchive(new Archive(Path.GetFileName(path), SEC13FEdgarDay.IndexUrl(day), day, day, IsDaily: true)); + FlushPendingRows(); + _edgarDays.Add(day); + read++; + } + + Log.Trace($"SEC13FDownloader.ProcessEdgarDays(): read {read} of {days.Count} EDGAR days"); + } + + /// + /// The data sets whose window ends before EDGAR takes over. A window that straddles the switch + /// would have its filings read from both sources or from neither, so it stops the run. + /// + internal static List ArchivesBefore(List archives, DateTime edgarFrom) + { + var straddling = archives.FirstOrDefault(archive => archive.Start < edgarFrom && archive.End >= edgarFrom); + if (straddling != null) + { + throw new InvalidOperationException( + $"SEC13FDownloader.ArchivesBefore(): {straddling.Name} covers {edgarFrom:yyyy-MM-dd}. EDGAR has to " + + "take over the day after a data set ends, or that window would be read twice or not at all."); + } + + return archives.Where(archive => archive.End < edgarFrom).ToList(); + } + + /// + /// The weekdays from one date to another that no earlier run has folded in, never before the + /// day EDGAR took over from the data sets. + /// + internal List EdgarDaysToRead(DateTime from, DateTime until) + { + var days = new List(); + var first = from.Date < _edgarFirstDay ? _edgarFirstDay : from.Date; + for (var day = first; day <= until.Date; day = day.AddDays(1)) + { + if (day.DayOfWeek != DayOfWeek.Saturday && day.DayOfWeek != DayOfWeek.Sunday && !_edgarDays.Contains(day)) + { + days.Add(day); + } + } + + return days; + } + + /// The last complete EDGAR day when the rebuild runs, in the SEC's time zone. + private static DateTime YesterdayInNewYork() + { + return DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).Date.AddDays(-1); + } + + /// + /// The moment a filing's rows are published: the day after its filing date at the release + /// time. A daily run that reads a day it had missed publishes it only after its own + /// deployment date, so the rows carry that instead of a time the run was not there for. + /// + internal DateTime ReleaseOf(DateTime filingDate) + { + var read = _deploymentDate.HasValue && _deploymentDate.Value.Date > filingDate.Date + ? _deploymentDate.Value.Date + : filingDate.Date; + return read + AvailableAfterFilingDate; + } + + /// + /// Stops a run into a destination that already holds files. The job hands it over empty, and + /// whatever an earlier run left there would be read into the universe and published again. + /// + internal void RequireEmptyDestination() + { + if (Directory.Exists(_destinationDirectory) && Directory.EnumerateFileSystemEntries(_destinationDirectory).Any()) + { + throw new InvalidOperationException( + $"SEC13FDownloader.Run(): {_destinationDirectory} is not empty. The destination has to start empty, " + + "or the files an earlier run left there would be published again."); + } + } + + /// + /// Stops an incremental run that cannot see the published history: merging into nothing would + /// republish thirteen years as one three month window and still report success. + /// + internal void RequirePublishedHistoryForIncrementalRun() + { + if (_deploymentDate == null) + { + return; + } + + var published = Directory.Exists(_processedDataDirectory) + ? Directory.EnumerateFiles(_processedDataDirectory, "*.csv").Count() + : 0; + + if (published == 0) + { + throw new InvalidOperationException( + $"SEC13FDownloader.Run(): the incremental run for {_deploymentDate:yyyy-MM-dd} found no published " + + $"history under {_processedDataDirectory}. Publishing this window on its own would truncate the " + + "dataset to it, so the run stops instead of reporting success."); + } + + Log.Trace($"SEC13FDownloader.Run(): {published} published securities to merge into, " + + $"read from {_processedDataDirectory}"); + + RequireFilerStateForIncrementalRun(); + RequireEdgarStateForIncrementalRun(); + } + + /// + /// Stops an incremental run that cannot tell which EDGAR days are already published: reading + /// one again would add its filings a second time under a later stamp. + /// + internal void RequireEdgarStateForIncrementalRun() + { + var path = Path.Combine(_processedDataDirectory, EdgarStateFileName); + if (File.Exists(path)) + { + return; + } + + throw new InvalidOperationException( + $"SEC13FDownloader.Run(): the incremental run for {_deploymentDate:yyyy-MM-dd} found no EDGAR state at " + + $"{path}, so it cannot tell a day already published from a new one. Republish the dataset with a full " + + "history run to restore it."); + } + + /// + /// Stops an incremental run without the filer state: every manager in tonight's archive would + /// look new and Holders would be inflated, with nothing else in the files to show it. + /// + internal void RequireFilerStateForIncrementalRun() + { + var path = Path.Combine(_processedDataDirectory, FilerStateFileName); + if (File.Exists(path)) + { + return; + } + + throw new InvalidOperationException( + $"SEC13FDownloader.Run(): the incremental run for {_deploymentDate:yyyy-MM-dd} found no filer state " + + $"at {path}. Holders is a distinct count of managers and cannot be rebuilt from the published totals, " + + "so every filer read tonight would be counted again on top of the existing history. The run stops " + + "instead of publishing an inflated count. Republish the dataset with a full history run to restore it."); + } + + /// + /// One archive: the zip name, its absolute URL, and the filing-receipt window it covers. A + /// daily one is a day read from EDGAR, which may hold no holdings filing at all. + /// + internal sealed record Archive(string Name, string Url, DateTime Start, DateTime End, bool IsDaily = false); + + /// + /// Scrapes the data sets page for every published archive, oldest first, so a new window is + /// picked up without a code change. + /// + private List GetArchives() + { + var html = GetWithRetry(DataSetsPageUrl); + var archives = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (Match match in ArchiveLinkRegex.Matches(html)) + { + var name = match.Groups["name"].Value; + if (archives.ContainsKey(name)) + { + continue; + } + + var href = WebUtility.HtmlDecode(match.Groups["href"].Value); + var url = href.StartsWith("http", StringComparison.OrdinalIgnoreCase) + ? href + : SecBaseUrl + (href.StartsWith("/", StringComparison.Ordinal) ? href : "/" + href); + + var (start, end) = ParseArchiveWindow(name); + archives[name] = new Archive(name, url, start, end); + } + + if (archives.Count == 0) + { + throw new InvalidOperationException( + $"SEC13FDownloader.GetArchives(): no _form13f.zip links found on {DataSetsPageUrl}. " + + "The page layout changed, or the request was blocked."); + } + + return archives.Values.OrderBy(x => x.End).ThenBy(x => x.Name, StringComparer.Ordinal).ToList(); + } + + /// + /// Reads the filing-receipt window out of an archive name. Two shapes are published: + /// "2013q2_form13f.zip" for the quarterly archives and "01mar2026-31may2026_form13f.zip" for + /// the rolling windows that replaced them. + /// + private static (DateTime start, DateTime end) ParseArchiveWindow(string name) + { + var quarter = QuarterArchiveRegex.Match(name); + if (quarter.Success) + { + var year = int.Parse(quarter.Groups["year"].Value, CultureInfo.InvariantCulture); + var number = int.Parse(quarter.Groups["quarter"].Value, CultureInfo.InvariantCulture); + var start = new DateTime(year, number * 3 - 2, 1); + return (start, start.AddMonths(3).AddDays(-1)); + } + + var window = WindowArchiveRegex.Match(name); + if (window.Success) + { + return (ParseWindowDate(window.Groups["start"].Value), ParseWindowDate(window.Groups["end"].Value)); + } + + throw new FormatException( + $"SEC13FDownloader.ParseArchiveWindow(): '{name}' matches neither the quarterly nor the " + + "rolling-window naming. A third naming scheme appeared and the selection would be wrong."); + } + + /// Parses a "01mar2026" style date out of a rolling-window archive name. + private static DateTime ParseWindowDate(string value) + { + return DateTime.ParseExact(value, "ddMMMyyyy", CultureInfo.InvariantCulture); + } + + /// + /// The first day EDGAR is responsible for: the day after the last data set the rebuild read. + /// Earlier days came from the data sets and are not in the list, so a daily run must never + /// read them from EDGAR; it did once, and counted their filings twice. + /// + private DateTime _edgarFirstDay; + + private const string EdgarFirstDayPrefix = "#from "; + + /// Loads the EDGAR days an earlier run published. A full run reads them all again and starts empty. + internal void ReadEdgarState() + { + if (_deploymentDate == null) + { + return; + } + + var path = Path.Combine(_processedDataDirectory, EdgarStateFileName); + var header = false; + foreach (var line in File.ReadLines(path)) + { + if (line.StartsWith(EdgarFirstDayPrefix, StringComparison.Ordinal)) + { + _edgarFirstDay = DateTime.ParseExact(line.Substring(EdgarFirstDayPrefix.Length).Trim(), + DateFormat.EightCharacter, CultureInfo.InvariantCulture); + header = true; + } + else if (!string.IsNullOrWhiteSpace(line)) + { + _edgarDays.Add(DateTime.ParseExact(line.Trim(), DateFormat.EightCharacter, CultureInfo.InvariantCulture)); + } + } + + if (!header) + { + throw new InvalidDataException($"SEC13FDownloader.ReadEdgarState(): {path} does not say which day EDGAR " + + "took over from the data sets, so a daily run could read a day twice. Republish with a full history run."); + } + + Log.Trace($"SEC13FDownloader.ReadEdgarState(): EDGAR from {_edgarFirstDay:yyyy-MM-dd}, {_edgarDays.Count} " + + $"days already published, the last {_edgarDays.Max:yyyy-MM-dd}"); + } + + /// Publishes the day EDGAR took over and the days folded in so far, oldest first. + internal void WriteEdgarState() + { + Directory.CreateDirectory(_destinationDirectory); + SEC13FFiles.WriteThenMove(Path.Combine(_destinationDirectory, EdgarStateFileName), stream => + { + using var writer = new StreamWriter(stream, leaveOpen: true); + writer.Write(EdgarFirstDayPrefix + _edgarFirstDay.ToString(DateFormat.EightCharacter, CultureInfo.InvariantCulture)); + writer.Write('\n'); + foreach (var day in _edgarDays) + { + writer.Write(day.ToString(DateFormat.EightCharacter, CultureInfo.InvariantCulture)); + writer.Write('\n'); + } + }); + } + + /// + /// Reads one archive end to end: the submissions that carry holdings, the confidential + /// treatment flags, and then the positions themselves, streamed rather than loaded. + /// + private void ProcessArchive(Archive archive) + { + // One window's worth of resolutions is all that is worth holding: the key carries the + // filing date, so entries from a window already read can never be hit again. + _resolvedCusips.Clear(); + _crosswalkResolutions.Clear(); + _admittedAmendments.Clear(); + _tickerOwners.Clear(); + + var path = DownloadArchive(archive); + + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read); + using var zip = new ZipArchive(stream, ZipArchiveMode.Read); + + var submissions = ReadSubmissions(zip, archive); + if (submissions.Count == 0 && !archive.IsDaily) + { + // The windows do not overlap, so an archive with no holdings means the layout moved, + // and carrying on would publish a history with this window missing. A single EDGAR + // day can legitimately carry none. + throw new InvalidDataException( + $"SEC13FDownloader.ProcessArchive(): {archive.Name} yielded no holdings submissions. " + + "An archive that contributes nothing means the upstream layout moved."); + } + + ApplyConfidentialFlags(zip, archive, submissions); + + var holdings = ReadInfoTable(zip, archive, submissions); + Log.Trace($"SEC13FDownloader.ProcessArchive(): {archive.Name}: {submissions.Count} submissions, " + + $"{holdings.Count} (security, period, release) groups"); + + EmitHoldings(holdings); + } + + private string DownloadArchive(Archive archive) + { + return DownloadFile(archive.Url, archive.Name, _archiveCacheDirectory); + } + + /// + /// Downloads a file into a directory, reusing a copy already there. It lands as ".part" first, + /// so an interrupted run cannot leave a truncated file for the next one. + /// + private string DownloadFile(string url, string name, string directory) + { + Directory.CreateDirectory(directory); + var path = Path.Combine(directory, name); + if (File.Exists(path)) + { + Log.Trace($"SEC13FDownloader.DownloadFile(): {name} already on disk"); + return path; + } + + var temporaryPath = path + ".part"; + WithRetry(url, () => + { + using var response = _client + .GetAsync(url, HttpCompletionOption.ResponseHeadersRead) + .GetAwaiter().GetResult(); + response.EnsureSuccessStatusCode(); + + using var source = response.Content.ReadAsStreamAsync().GetAwaiter().GetResult(); + using var destination = new FileStream(temporaryPath, FileMode.Create, FileAccess.Write); + source.CopyTo(destination); + return true; + }); + + File.Move(temporaryPath, path, overwrite: true); + Log.Trace($"SEC13FDownloader.DownloadFile(): {name}, {new FileInfo(path).Length} bytes"); + return path; + } + + /// + /// Whether the SEC has published a file: false only on 404. Anything else fails once the + /// retries run out, since a 403 taken for "not published" would change the crosswalk quietly. + /// + private bool UrlExists(string url) + { + return WithRetry(url, () => + { + using var request = new HttpRequestMessage(HttpMethod.Head, url); + using var response = _client.Send(request); + if (response.StatusCode == HttpStatusCode.NotFound) + { + return false; + } + + response.EnsureSuccessStatusCode(); + return true; + }); + } + + /// One filing that carries holdings, keyed in the caller by accession number. + internal sealed class Submission + { + public int Cik { get; init; } + public DateTime FilingDate { get; init; } + public DateTime Period { get; init; } + public bool ConfidentialOmitted { get; set; } + + /// True for a 13F-HR/A, which restates a report this filer already made. + public bool IsAmendment { get; init; } + } + + /// + /// Reads SUBMISSION.tsv, keeping only the filings that carry positions and that no earlier + /// archive already contributed. + /// + private Dictionary ReadSubmissions(ZipArchive zip, Archive archive) + { + var submissions = new Dictionary(StringComparer.Ordinal); + var notices = 0; + + foreach (var (columns, fields) in ReadTable(zip, archive, "SUBMISSION.tsv", + "ACCESSION_NUMBER", "FILING_DATE", "SUBMISSIONTYPE", "CIK", "PERIODOFREPORT")) + { + var submissionType = fields[columns["SUBMISSIONTYPE"]]; + if (submissionType.StartsWith(NoticeSubmissionTypePrefix, StringComparison.OrdinalIgnoreCase)) + { + notices++; + continue; + } + + var accession = fields[columns["ACCESSION_NUMBER"]]; + if (!_processedAccessions.Add(accession)) + { + continue; + } + + submissions[accession] = new Submission + { + Cik = int.Parse(fields[columns["CIK"]], NumberStyles.Integer, CultureInfo.InvariantCulture), + FilingDate = ParseSecDate(fields[columns["FILING_DATE"]], archive, "FILING_DATE"), + Period = ParseSecDate(fields[columns["PERIODOFREPORT"]], archive, "PERIODOFREPORT"), + IsAmendment = submissionType.EndsWith(AmendmentSubmissionTypeSuffix, StringComparison.OrdinalIgnoreCase) + }; + } + + Log.Trace($"SEC13FDownloader.ReadSubmissions(): {archive.Name}: {submissions.Count} holdings filings, " + + $"{notices} notices skipped"); + return submissions; + } + + /// + /// Marks the filings that withheld positions under confidential treatment (SUMMARYPAGE + /// ISCONFIDENTIALOMITTED). COVERPAGE CONFDENIEDEXPIRED is not read: it means the opposite, + /// positions withheld before and disclosed in this filing. + /// + private void ApplyConfidentialFlags(ZipArchive zip, Archive archive, Dictionary submissions) + { + var flagged = 0; + + foreach (var (columns, fields) in ReadTable( + zip, archive, "SUMMARYPAGE.tsv", "ACCESSION_NUMBER", "ISCONFIDENTIALOMITTED")) + { + if (!IsYes(fields[columns["ISCONFIDENTIALOMITTED"]]) || + !submissions.TryGetValue(fields[columns["ACCESSION_NUMBER"]], out var submission) || + submission.ConfidentialOmitted) + { + continue; + } + + submission.ConfidentialOmitted = true; + flagged++; + } + + Log.Trace($"SEC13FDownloader.ApplyConfidentialFlags(): {archive.Name}: {flagged} filings flagged"); + } + + /// The summed measures of a set of reported lines. + internal class Measures + { + public decimal Shares { get; set; } + public decimal HoldingValue { get; set; } + public decimal CallShares { get; set; } + public decimal PutShares { get; set; } + public decimal PrincipalValue { get; set; } + public decimal VotingSole { get; set; } + public decimal VotingShared { get; set; } + public bool ConfidentialOmitted { get; set; } + + /// Adds another set of lines into this one. + public void Add(Measures other) + { + Shares += other.Shares; + HoldingValue += other.HoldingValue; + CallShares += other.CallShares; + PutShares += other.PutShares; + PrincipalValue += other.PrincipalValue; + VotingSole += other.VotingSole; + VotingShared += other.VotingShared; + ConfidentialOmitted |= other.ConfidentialOmitted; + } + + /// The published value columns, Holders first. + public decimal[] ToValues(int holders) + { + return new[] { holders, Shares, HoldingValue, CallShares, PutShares, PrincipalValue, VotingSole, VotingShared }; + } + } + + /// + /// The reported lines sharing one (CUSIP, period, release). The measures hold the original + /// filings; amending filers' lines wait in Amendments for AdmitAmendment. + /// + internal sealed class Holding : Measures + { + /// Distinct filer CIKs of the original filings, which is what Holders counts. + public HashSet Ciks { get; } = new(); + + /// Each amending filer's lines, by CIK. + public Dictionary Amendments { get; } = new(); + + /// Every reported value, originals and amendments, for the coverage summary. + public decimal ReportedValue => HoldingValue + Amendments.Values.Sum(amendment => amendment.HoldingValue); + + /// + /// The implied price, VALUE over SSHPRNAMT as reported, of every share line in the group, + /// which is what a crosswalk resolution is checked against the close with. + /// + public List SharePrices { get; } = new(); + } + + /// + /// Every (security, quarter, filer) already counted, so a filer reaches Holders once. Packed + /// into a long (security index in bits 42..61, period index 32..41, CIK 0..31) because the + /// history holds 58 million of them. The value is the filing date first counted on, as + /// yyyyMMdd, kept in the published state for whoever needs to tell when a filer arrived. + /// + private readonly Dictionary _countedFilers = new(); + + private readonly Dictionary _securityIndexes = new(); + private readonly List _securitiesByIndex = new(); + private readonly Dictionary _periodIndexes = new(); + private readonly List _periodsByIndex = new(); + + private const int SecurityIndexBits = 20; + private const int PeriodIndexBits = 10; + + /// Packs a counted filer into its key, assigning either index on first sight. + private long FilerKey(SecurityIdentifier security, DateTime period, int cik) + { + if (!_securityIndexes.TryGetValue(security, out var securityIndex)) + { + securityIndex = _securitiesByIndex.Count; + if (securityIndex >= 1 << SecurityIndexBits) + { + // Throwing rather than wrapping: a wrapped index would silently credit one + // security's filers to another, and no row count would show it. + throw new InvalidOperationException( + $"More than {1 << SecurityIndexBits} securities; the filer key needs more bits"); + } + + _securityIndexes[security] = securityIndex; + _securitiesByIndex.Add(security); + } + + if (!_periodIndexes.TryGetValue(period, out var periodIndex)) + { + periodIndex = _periodsByIndex.Count; + if (periodIndex >= 1 << PeriodIndexBits) + { + throw new InvalidOperationException( + $"More than {1 << PeriodIndexBits} reported periods; the filer key needs more bits"); + } + + _periodIndexes[period] = periodIndex; + _periodsByIndex.Add(period); + } + + return ((long)securityIndex << 42) | ((long)periodIndex << 32) | (uint)cik; + } + + /// + /// How many of these filers had not been counted for this security and quarter yet, marking + /// them counted. A manager reaches Holders once, on the first day it reports the security. + /// + internal int CountNewFilers(SecurityIdentifier security, DateTime period, IEnumerable ciks, + DateTime filingDate) + { + var stamp = int.Parse(filingDate.ToStringInvariant(DateFormat.EightCharacter), CultureInfo.InvariantCulture); + var newFilers = 0; + + foreach (var cik in ciks) + { + if (_countedFilers.TryAdd(FilerKey(security, period, cik), stamp)) + { + newFilers++; + } + } + + return newFilers; + } + + /// Swaps a filer key's security and period indexes through the given maps, keeping the CIK. + private static long Repack(long key, int[] securities, int[] periods) + { + var securityIndex = (int)(key >> 42); + var periodIndex = (int)((key >> 32) & ((1 << PeriodIndexBits) - 1)); + return ((long)securities[securityIndex] << 42) | ((long)periods[periodIndex] << 32) | (key & 0xFFFFFFFFL); + } + + /// The inverse of a permutation: for each value, the position it holds. + private static int[] Invert(int[] order) + { + var positions = new int[order.Length]; + for (var position = 0; position < order.Length; position++) + { + positions[order[position]] = position; + } + + return positions; + } + + /// Unpacks a filer key back into the three values the state file carries. + private (SecurityIdentifier Security, DateTime Period, int Cik) FilerParts(long key) + { + var securityIndex = (int)(key >> 42); + var periodIndex = (int)((key >> 32) & ((1 << PeriodIndexBits) - 1)); + return (_securitiesByIndex[securityIndex], _periodsByIndex[periodIndex], (int)(uint)key); + } + + /// + /// The counted filers, published next to the security files so an incremental run can tell a + /// filer already counted from a new one: a distinct count cannot be rebuilt from the totals. + /// + private const string FilerStateFileName = "filers.zip"; + + private const string FilerStateEntryName = "filers.csv"; + + /// The stamp written on the state entry, fixed so the archive is byte reproducible. + private static readonly DateTimeOffset FilerStateTimestamp = + new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + + /// + /// Loads the counted filers an earlier run published. A full run rebuilds every count and + /// starts empty. + /// + internal void ReadFilerState() + { + if (_deploymentDate == null) + { + Log.Trace("SEC13FDownloader.ReadFilerState(): full history run, rebuilding the filer state from scratch"); + return; + } + + var path = Path.Combine(_processedDataDirectory, FilerStateFileName); + using var file = File.OpenRead(path); + using var zip = new ZipArchive(file, ZipArchiveMode.Read); + + var entry = zip.GetEntry(FilerStateEntryName) + ?? throw new InvalidOperationException($"{path} carries no {FilerStateEntryName}"); + + using var reader = new StreamReader(entry.Open()); + string line; + var rows = 0; + + while ((line = reader.ReadLine()) != null) + { + if (line.Length == 0) + { + continue; + } + + var parts = line.Split(','); + if (parts.Length != 4) + { + throw new InvalidOperationException($"{path}: '{line}' is not a filer row"); + } + + _countedFilers[FilerKey( + SecurityIdentifier.Parse(parts[0]), + DateTime.ParseExact(parts[1], DateFormat.EightCharacter, CultureInfo.InvariantCulture), + int.Parse(parts[2], CultureInfo.InvariantCulture))] = int.Parse(parts[3], CultureInfo.InvariantCulture); + rows++; + } + + Log.Trace($"SEC13FDownloader.ReadFilerState(): read {rows} counted filers from {path}, " + + $"{_securitiesByIndex.Count} securities and {_periodsByIndex.Count} periods"); + } + + /// + /// Publishes the counted filers sorted by security, quarter and CIK. The order comes from the + /// values and not from the packed keys, whose indexes depend on the order things were first + /// seen, so a full run and an incremental one write the same bytes. + /// + internal void WriteFilerState() + { + Directory.CreateDirectory(_destinationDirectory); + + var path = Path.Combine(_destinationDirectory, FilerStateFileName); + + var securityOrder = Enumerable.Range(0, _securitiesByIndex.Count) + .OrderBy(index => _securitiesByIndex[index].ToString(), StringComparer.Ordinal) + .ToArray(); + var periodOrder = Enumerable.Range(0, _periodsByIndex.Count) + .OrderBy(index => _periodsByIndex[index]) + .ToArray(); + var securityRanks = Invert(securityOrder); + var periodRanks = Invert(periodOrder); + + var keys = new long[_countedFilers.Count]; + var next = 0; + foreach (var key in _countedFilers.Keys) + { + keys[next++] = Repack(key, securityRanks, periodRanks); + } + + Array.Sort(keys); + + SEC13FFiles.WriteThenMove(path, file => + { + using var zip = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true); + var entry = zip.CreateEntry(FilerStateEntryName, CompressionLevel.Optimal); + + // A fixed stamp, or the same filers would hash differently on every run. + entry.LastWriteTime = FilerStateTimestamp; + + using var writer = new StreamWriter(entry.Open()); + foreach (var ranked in keys) + { + var key = Repack(ranked, securityOrder, periodOrder); + var (security, period, cik) = FilerParts(key); + writer.Write(security.ToString()); + writer.Write(','); + writer.Write(period.ToStringInvariant(DateFormat.EightCharacter)); + writer.Write(','); + writer.Write(cik.ToStringInvariant()); + writer.Write(','); + writer.Write(_countedFilers[key].ToStringInvariant()); + writer.Write('\n'); + } + }); + Log.Trace($"SEC13FDownloader.WriteFilerState(): wrote {keys.Length} counted filers to {path}, " + + $"{new FileInfo(path).Length / (1024 * 1024)} MB"); + } + + /// The group a reported line belongs to. + internal readonly record struct HoldingKey(string Cusip, DateTime Period, DateTime FilingDate); + + /// + /// Streams INFOTABLE.tsv and sums every reported line into its group, with no dedup by + /// discretion or other manager: measured against shares outstanding the raw sum matches + /// published ownership, and both dedup rules understate it about three times. + /// + internal Dictionary ReadInfoTable( + ZipArchive zip, Archive archive, Dictionary submissions) + { + var holdings = new Dictionary(); + var units = DetectValueUnits(zip, archive, submissions); + + foreach (var (columns, fields) in ReadTable(zip, archive, "INFOTABLE.tsv", + "ACCESSION_NUMBER", "CUSIP", "VALUE", "SSHPRNAMT", "SSHPRNAMTTYPE", "PUTCALL", + "VOTING_AUTH_SOLE", "VOTING_AUTH_SHARED")) + { + var accession = fields[columns["ACCESSION_NUMBER"]]; + if (!submissions.TryGetValue(accession, out var submission)) + { + // A notice, or a filing an earlier archive already contributed. + continue; + } + + var cusip = fields[columns["CUSIP"]].Trim().ToUpperInvariant(); + if (cusip.Length == 0) + { + continue; + } + + var key = new HoldingKey(cusip, submission.Period, submission.FilingDate); + if (!holdings.TryGetValue(key, out var group)) + { + holdings[key] = group = new Holding(); + } + + // Holders counts every manager that reported the security, options included. An + // amendment's lines are held apart per filer until AdmitAmendment knows whether the + // manager had already reported it. + Measures holding; + if (submission.IsAmendment) + { + if (!group.Amendments.TryGetValue(submission.Cik, out holding)) + { + group.Amendments[submission.Cik] = holding = new Measures(); + } + } + else + { + group.Ciks.Add(submission.Cik); + holding = group; + } + + holding.ConfidentialOmitted |= submission.ConfidentialOmitted; + + var shareType = fields[columns["SSHPRNAMTTYPE"]].Trim(); + var putCall = fields[columns["PUTCALL"]].Trim(); + var amount = ParseDecimal(fields[columns["SSHPRNAMT"]]); + + if (putCall.Equals("Call", StringComparison.OrdinalIgnoreCase)) + { + holding.CallShares += amount; + continue; + } + + if (putCall.Equals("Put", StringComparison.OrdinalIgnoreCase)) + { + holding.PutShares += amount; + continue; + } + + // On a PRN line SSHPRNAMT is the principal amount, which is what PrincipalValue + // publishes; VALUE is the market value of that debt. + if (shareType.Equals("PRN", StringComparison.OrdinalIgnoreCase)) + { + holding.PrincipalValue += amount; + continue; + } + + if (!shareType.Equals("SH", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + // VALUE only counts under the share filter: unfiltered it totals $70.1 trillion for a + // single quarter. In whole dollars, whichever unit the line used. + var value = ParseDecimal(fields[columns["VALUE"]]); + if (value > 0m && amount > 0m) + { + group.SharePrices.Add((double)(value / amount)); + } + + holding.Shares += amount; + holding.HoldingValue += value * units.Factor(accession, cusip, submission, value, amount); + holding.VotingSole += ParseDecimal(fields[columns["VOTING_AUTH_SOLE"]]); + holding.VotingShared += ParseDecimal(fields[columns["VOTING_AUTH_SHARED"]]); + } + + Log.Trace($"SEC13FDownloader.ReadInfoTable(): {archive.Name}: {units.CorrectedLines} share lines in " + + "a different unit than the rest of their filing"); + return holdings; + } + + /// + /// The factors that turn VALUE into whole dollars. Filers do not all follow the 2023 unit + /// change (in Apple's December 2019 quarter 85 lines in dollars made 88 percent of the scaled + /// total), and some mix units within one filing, so each implied price is compared with the + /// security's close on the quarter's last trading day: per line where the close is known, and + /// per filing, from its own lines, where it is not. Nothing here reads another filing, so a + /// filing comes out the same read alone on its day as inside a three month window. The + /// median of every filer in the window it replaces corrected a filing with others made weeks + /// after it. + /// + internal ValueUnits DetectValueUnits(ZipArchive zip, Archive archive, Dictionary submissions) + { + var offsetsByFiling = new Dictionary>(StringComparer.Ordinal); + var pricedByFiling = new Dictionary(StringComparer.Ordinal); + + foreach (var (columns, fields) in ReadTable(zip, archive, "INFOTABLE.tsv", + "ACCESSION_NUMBER", "CUSIP", "VALUE", "SSHPRNAMT", "SSHPRNAMTTYPE", "PUTCALL")) + { + var accession = fields[columns["ACCESSION_NUMBER"]]; + if (!submissions.TryGetValue(accession, out var submission) + || !fields[columns["SSHPRNAMTTYPE"]].Trim().Equals("SH", StringComparison.OrdinalIgnoreCase) + || fields[columns["PUTCALL"]].Trim().Length > 0) + { + continue; + } + + var offset = MarketOffset(fields[columns["CUSIP"]].Trim().ToUpperInvariant(), submission, + ParseDecimal(fields[columns["VALUE"]]), ParseDecimal(fields[columns["SSHPRNAMT"]])); + if (offset == null) + { + continue; + } + + pricedByFiling[accession] = pricedByFiling.GetValueOrDefault(accession) + 1; + + // A price on no unit step says nothing about the unit the filing reports in. + if (UnitStep(offset.Value) == null) + { + continue; + } + + if (!offsetsByFiling.TryGetValue(accession, out var offsets)) + { + offsetsByFiling[accession] = offsets = new List(); + } + + offsets.Add(offset.Value); + } + + var units = new Dictionary(StringComparer.Ordinal); + var againstTheRule = 0; + foreach (var (accession, submission) in submissions) + { + var thousandsRule = submission.FilingDate < ValueInWholeDollarsFrom; + units[accession] = ValueMultiplier( + offsetsByFiling.TryGetValue(accession, out var offsets) ? offsets : new List(), + pricedByFiling.GetValueOrDefault(accession), thousandsRule); + if (units[accession] != (thousandsRule ? 1000m : 1m)) + { + againstTheRule++; + } + } + + // Filings with closes to compare against whose lines mostly sit on no unit step. + var broken = new HashSet( + pricedByFiling + .Where(pair => !IsUnitEvidence(offsetsByFiling.GetValueOrDefault(pair.Key)?.Count ?? 0, pair.Value)) + .Select(pair => pair.Key), + StringComparer.Ordinal); + + Log.Trace($"SEC13FDownloader.DetectValueUnits(): {archive.Name}: {againstTheRule} of " + + $"{submissions.Count} filings report VALUE in the other unit, {broken.Count} show no unit at all"); + + return new ValueUnits(units, broken, MarketOffset); + } + + /// + /// log10 of a share line's implied price over its security's quarter-end close, or null when + /// the line has no price, the CUSIP resolves to nothing, or the close is not known. Near zero + /// the line is in whole dollars, near -3 in thousands. + /// + internal double? MarketOffset(string cusip, Submission submission, decimal value, decimal amount) + { + if (value <= 0m || amount <= 0m) + { + return null; + } + + var security = ResolveSecurity(cusip, submission.FilingDate); + var close = security == null ? null : _closePrices.Close(security, submission.Period, submission.FilingDate); + return close == null ? null : Math.Log10((double)(value / amount / close.Value)); + } + + /// + /// The whole-dollar factor of each share line: its own, measured against its security's close + /// when that is known, or else the one decided for its whole filing. + /// + internal sealed class ValueUnits + { + private readonly Dictionary _filings; + private readonly HashSet _broken; + private readonly Func _marketOffset; + + /// Share lines whose factor differed from their filing's. + public long CorrectedLines { get; private set; } + + public ValueUnits(Dictionary filings, HashSet broken, + Func marketOffset) + { + _filings = filings; + _broken = broken; + _marketOffset = marketOffset; + } + + /// The factor for one share line of VALUE over shares. + public decimal Factor(string accession, string cusip, Submission submission, decimal value, decimal amount) + { + // A filing whose lines mostly sit on no unit step keeps its unscaled factor on every + // line: the few that land on a step do so by chance. + var filing = _filings[accession]; + if (_broken.Contains(accession)) + { + return filing; + } + + var offset = _marketOffset(cusip, submission, value, amount); + if (offset == null) + { + return filing; + } + + // The thousandfold step the line's price sits on: one below is VALUE in thousands, one + // above is VALUE typed a thousand times too large. That second case is a VALUE slip + // rather than a share count one: of the 4,740 such lines of the March 2023 quarter + // whose filer reported the same security the quarter before, 4,578 held the same + // shares then and 162 a thousand times more. A price on no step, a millionfold one + // included, cannot be checked, so it takes the smaller of its filing's unit and the rule + // of its filing date. Either one alone inflates an era: the rule scaled the odd lines of + // filers in whole dollars before 2023 ($730 billion in the September 2022 quarter), the + // filing's unit those of filers still in thousands in early 2023 ($879 billion in + // December 2022). Scaling by the nearest step put Seagate at $413 billion. + var step = UnitStep(offset.Value); + if (step == null) + { + return Math.Min(filing, submission.FilingDate < ValueInWholeDollarsFrom ? 1000m : 1m); + } + + var factor = step == 0 ? 1m : step < 0 ? 1000m : 0.001m; + if (factor != filing) + { + CorrectedLines++; + } + + return factor; + } + } + + /// + /// Whether a filing's VALUE is multiplied by a thousand. The offsets are log10 of those of its + /// implied prices that sit on a unit step of the security's close, out of + /// lines with a close: near zero the filing reports whole dollars, + /// near -3 thousands. They say so only when they are most of its priced lines. A filing whose + /// lines mostly sit on no step is broken rather than in another unit, and the few that land on + /// one do so by chance: a manager that typed its dollar values as share counts, a price of $1 + /// on every line, looks like thousands against any close near $1,000. Such a filing is never + /// scaled up: before 2023 the rule multiplied it by a thousand, and one manager whose share + /// counts carried VALUE in thousands, $1,000 a share on every line, put Alphabet's September + /// 2020 quarter 9.5 percent above its close. With nothing to compare against, the rule stands. + /// + internal static decimal ValueMultiplier(List offsets, int priced, bool thousandsRule) + { + if (!IsUnitEvidence(offsets.Count, priced)) + { + return priced == 0 && thousandsRule ? 1000m : 1m; + } + + return Median(offsets) < -1.5 ? 1000m : 1m; + } + + /// Whether a filing's lines on a unit step are most of its priced lines. + private static bool IsUnitEvidence(int onAStep, int priced) + { + return priced > 0 && onAStep * 2 > priced; + } + + private static double Median(List values) + { + var sorted = values.OrderBy(value => value).ToList(); + var middle = sorted.Count / 2; + return sorted.Count % 2 == 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; + } + + /// + /// Resolves each group to a security and queues its increment, the filings of that day only; + /// the finalize pass turns increments into running totals. Groups that resolve to nothing, + /// mostly foreign issuers, are dropped. + /// + private void EmitHoldings(Dictionary holdings) + { + // Filing date order, since a filer counts on the first day it reports the security. + foreach (var day in holdings.GroupBy(entry => entry.Key.FilingDate).OrderBy(group => group.Key)) + { + var resolved = new List<(HoldingKey Key, Holding Holding, SecurityIdentifier Security, string Ticker, int Counted)>(); + + // Originals first, for every group of the day, so an amendment filed the same day as + // the manager's original is recognised as a restatement whichever CUSIP comes first. + foreach (var (key, holding) in day) + { + var security = ResolveSecurity(key.Cusip, key.FilingDate); + var ticker = security == null ? null : ResolveTicker(security, key.FilingDate); + if (string.IsNullOrWhiteSpace(ticker) || !IsFileNameSafe(ticker)) + { + _unresolvedGroups++; + _unresolvedValue += holding.ReportedValue; + continue; + } + + // Whoever reads the ticker's file takes its rows to belong to the security that + // owns the ticker that day; a group of another security is dropped, not mixed in. + var owner = TickerOwner(ticker, key.FilingDate); + if (owner != security.ToString()) + { + _conflictingTickers++; + _conflictingValue += holding.ReportedValue; + if (_conflictSample.Count < 10) + { + _conflictSample.Add($"{key.Cusip} {key.FilingDate:yyyy-MM-dd} {ticker} is {security}, owner {owner ?? "none"}"); + } + + _unresolvedGroups++; + _unresolvedValue += holding.ReportedValue; + continue; + } + + // A crosswalk ticker is a fund administrator's free text, and a wrong one names + // another company: DeFi Technologies at $2 reached the $129 Hashdex DEFI ETF and + // gave it 111 holders. A group whose prices say so is dropped. + if (_crosswalkResolutions.Contains((key.Cusip, key.FilingDate)) && + !KeepsCrosswalkGroup(key.Cusip, security, key.Period, key.FilingDate, holding.SharePrices)) + { + if (IsEquityIssue(key.Cusip)) + { + _mismatchedGroups++; + } + else + { + _debtCusipsRejected++; + } + + _mismatchedValue += holding.ReportedValue; + _unresolvedGroups++; + _unresolvedValue += holding.ReportedValue; + continue; + } + + // Counted after the guards above: a group that resolves to nothing must not + // consume the filer, or the day it does resolve would find it already taken. + resolved.Add((key, holding, security, ticker.ToLowerInvariant(), + CountNewFilers(security, key.Period, holding.Ciks, key.FilingDate))); + } + + foreach (var (key, holding, security, ticker, counted) in resolved) + { + var admitted = new Measures(); + admitted.Add(holding); + var newFilers = counted; + + foreach (var (cik, amendment) in holding.Amendments) + { + if (AdmitAmendment(security, key.Period, cik, key.FilingDate, out var isNew)) + { + admitted.Add(amendment); + newFilers += isNew ? 1 : 0; + } + } + + _resolvedValue += holding.ReportedValue; + + // A day of nothing but restatements of counted positions changes no total. + var values = admitted.ToValues(newFilers); + if (!admitted.ConfidentialOmitted && values.All(value => value == 0m)) + { + _unchangedGroups++; + continue; + } + + Queue(security.ToString(), ticker, new HoldingsRow + { + Time = ReleaseOf(key.FilingDate), + PeriodEnd = key.Period, + Values = values, + ConfidentialOmitted = admitted.ConfidentialOmitted + }); + } + } + } + + /// + /// Whether an amendment's lines for this security and quarter are added, and whether its + /// filer is new: only when the manager had not reported the security for the quarter yet. + /// + internal bool AdmitAmendment(SecurityIdentifier security, DateTime period, int cik, DateTime filingDate, + out bool newFiler) + { + var key = FilerKey(security, period, cik); + var stamp = int.Parse(filingDate.ToStringInvariant(DateFormat.EightCharacter), CultureInfo.InvariantCulture); + + if (_countedFilers.TryAdd(key, stamp)) + { + _admittedAmendments.Add((key, stamp)); + newFiler = true; + return true; + } + + // Already counted: either by this same amendment through another CUSIP of the same + // security, which is still part of that one new position, or by an earlier report, + // which this amendment restates. + newFiler = false; + return _admittedAmendments.Contains((key, stamp)); + } + + /// + /// The security trading under a ticker on a date per the map files, or null. It is how the + /// universe and LEAN read a ticker file, so writing applies the same test. + /// + private string TickerOwner(string ticker, DateTime date) + { + var key = (ticker.ToUpperInvariant(), date.Date); + if (!_tickerOwners.TryGetValue(key, out var owner)) + { + var mapFile = _mapFileProvider + .Get(new AuxiliaryDataKey(Market.USA, SecurityType.Equity)) + .ResolveMapFile(key.Item1, key.Item2); + + _tickerOwners[key] = owner = mapFile.Any() + ? SecurityIdentifier.GenerateEquity(mapFile.FirstDate, mapFile.FirstTicker, Market.USA).ToString() + : null; + } + + return owner; + } + + /// + /// Resolves a CUSIP to a security, point in time: by CUSIP, then by the US ISIN built from + /// it, then through the N-PORT ticker crosswalk. + /// + internal SecurityIdentifier ResolveSecurity(string rawCusip, DateTime tradingDate) + { + var key = (rawCusip, tradingDate); + if (_resolvedCusips.TryGetValue(key, out var cached)) + { + return cached; + } + + // The summary counts CUSIPs, not lookups, and one CUSIP is now resolved once per filing + // date rather than once per run. + var firstSighting = _countedCusips.Add(rawCusip); + + var cusip = NormalizeCusip(rawCusip); + if (cusip == null) + { + if (firstSighting) + { + _malformedCusips++; + } + + _resolvedCusips[key] = null; + return null; + } + + // Step 1: LEAN stores the CUSIP without its check digit, so the ninth character comes off. + var isin = BuildUnitedStatesIsin(cusip); + var security = TradingDefinition(_definitionsByCusip, cusip.Substring(0, 8), isin, tradingDate); + if (security != null) + { + if (firstSighting) + { + _resolvedByCusip++; + } + } + else + { + // Step 2: the US ISIN reaches issuers whose CUSIP is blank in the security database. + security = TradingDefinition(_definitionsByIsin, isin, isin, tradingDate); + if (security != null && firstSighting) + { + _resolvedByIsin++; + } + } + + // Step 3: the N-PORT crosswalk needs no security database, and reaches Alphabet and the + // CINS foreign issuers the constructed ISIN cannot represent. + if (security == null) + { + var identifier = ResolveThroughTicker(cusip); + if (identifier != null) + { + if (firstSighting) + { + _resolvedByTicker++; + } + + _resolvedCusips[key] = identifier; + _crosswalkResolutions.Add(key); + return identifier; + } + + _unresolvedCusips.Add(cusip); + _resolvedCusips[key] = null; + return null; + } + + _resolvedCusips[key] = security; + return security; + } + + /// + /// The security, among the database rows carrying an identifier, that trades under its own + /// ticker on the date, or null. The database repeats identifiers across the listings one + /// company has had, and LEAN's resolver takes the first row, mostly the one that no longer + /// trades: Alcoa's CUSIP sits on the old Alcoa, now Howmet, and on the Alcoa spun off in 2016, + /// and TG Therapeutics' on the listing it had as Atlantic Technology Ventures. Run on the real + /// database that dropped Alcoa, Howmet, Vertiv and TG Therapeutics, as groups whose ticker + /// another security owned. A row carrying the ISIN the CUSIP builds is tried first, since the + /// old Alcoa row carries Howmet's. + /// + private SecurityIdentifier TradingDefinition(Dictionary> rowsByIdentifier, + string identifier, string isin, DateTime tradingDate) + { + if (!rowsByIdentifier.TryGetValue(identifier, out var rows)) + { + return null; + } + + int Priority(SecurityDefinition row) => row.ISIN == null ? 1 + : string.Equals(row.ISIN, isin, StringComparison.OrdinalIgnoreCase) ? 0 : 2; + + foreach (var row in rows.OrderBy(Priority)) + { + var ticker = ResolveTicker(row.SecurityIdentifier, tradingDate); + if (ticker != null && TickerOwner(ticker, tradingDate) == row.SecurityIdentifier.ToString()) + { + return row.SecurityIdentifier; + } + } + + return null; + } + + /// + /// The security database rows by one identifier, in file order. A CUSIP is keyed by its eight + /// character body, the form the database mostly stores; the few rows written with their check + /// digit are keyed only when that digit holds. + /// + private static Dictionary> IndexDefinitions( + IEnumerable definitions, Func identifier) + { + var index = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var definition in definitions) + { + var key = identifier(definition); + if (string.IsNullOrWhiteSpace(key)) + { + continue; + } + + if (!index.TryGetValue(key, out var rows)) + { + index[key] = rows = new List(); + } + + rows.Add(definition); + } + + return index; + } + + /// The eight character body of a database CUSIP, or null when it is neither that nor a checked nine. + private static string DatabaseCusip(string cusip) + { + if (cusip == null) + { + return null; + } + + return cusip.Length switch + { + 8 => cusip, + 9 when ComputeCusipCheckDigit(cusip.Substring(0, 8)) == cusip[8] => cusip.Substring(0, 8), + _ => null + }; + } + + /// + /// Brings a reported CUSIP to nine characters, or null when it cannot be one. A short value + /// has lost either its leading zero ("37833100", Apple) or its check digit ("46428722", an + /// iShares fund). Padding the second with a zero would invent another security, so the + /// check digit decides which repair applies. + /// + internal static string NormalizeCusip(string cusip) + { + if (cusip.Length == 9) + { + return cusip; + } + + if (cusip.Length is < 6 or > 9) + { + return null; + } + + var padded = cusip.PadLeft(9, '0'); + if (ComputeCusipCheckDigit(padded.Substring(0, 8)) == padded[8]) + { + return padded; + } + + if (cusip.Length != 8) + { + return null; + } + + var checkDigit = ComputeCusipCheckDigit(cusip); + return checkDigit.HasValue ? cusip + checkDigit.Value : null; + } + + /// + /// The CUSIP check digit: a modulus ten sum over the eight character body where every + /// second character counts double, letters standing for their position in the alphabet + /// plus nine. + /// + internal static char? ComputeCusipCheckDigit(string body) + { + var sum = 0; + for (var i = 0; i < body.Length; i++) + { + var character = body[i]; + int value; + if (char.IsDigit(character)) + { + value = character - '0'; + } + else if (char.IsLetter(character)) + { + value = char.ToUpperInvariant(character) - 'A' + 10; + } + else + { + value = character switch { '*' => 36, '@' => 37, '#' => 38, _ => -1 }; + if (value < 0) + { + // Not a character a CUSIP can hold, so there is no check digit. + return null; + } + } + + if (i % 2 == 1) + { + value *= 2; + } + + sum += value / 10 + value % 10; + } + + return (char)('0' + (10 - sum % 10) % 10); + } + + /// + /// Priced share lines a crosswalk group needs before its prices can overrule the resolution. One + /// or two lines are usually one manager's slip, a stale price or a split, not another company: + /// on the December 2025 quarter, dropping on any disagreement removed 14,696 holders across + /// 3,249 tickers besides the misattributed ones, and requiring three lines halves that while + /// still removing 565 of the 637 misattributed holders. + /// + private const int PricedLinesToOverrule = 3; + + /// + /// Whether a group reached through the crosswalk stays. A CUSIP whose issue number carries + /// letters is debt as a rule, and N-PORT tags a fund's bonds with the issuer's ticker, so + /// Etsy's convertible notes reached Etsy's stock: 171 million of the 298 million shares its + /// December 2022 quarter published. Such a group stays only when its prices are the security's + /// own, which is how the iShares iBonds ETFs, whose CUSIPs carry letters too, keep their + /// holders. Any other group is dropped only when enough of its prices say it is another + /// company, as Centerra Gold and Enerflex reached Carlyle and Equifax through their Toronto + /// tickers CG and EFX. A group without prices, a day of option positions only, or a security + /// the close file does not carry, cannot be checked and stays. + /// + internal bool KeepsCrosswalkGroup(string cusip, SecurityIdentifier security, DateTime period, DateTime filingDate, + List prices) + { + var matches = PricesMatchClose(security, period, filingDate, prices); + if (!IsEquityIssue(cusip)) + { + return matches == true; + } + + return !(matches == false && prices.Count >= PricedLinesToOverrule); + } + + /// + /// Whether a group's prices are the security's: at least half of them sit on a unit step of the + /// quarter-end close. Counted line by line rather than through a median, because a busy day + /// mixes managers in dollars and in thousands: Avanos on 5 February 2026 had four lines at + /// $0.0112 and four at $11.23, and their median sat half way, on neither. Null when the group + /// has no priced share line or the security has no close. + /// + private bool? PricesMatchClose(SecurityIdentifier security, DateTime period, DateTime filingDate, List prices) + { + var close = _closePrices.Close(security, period, filingDate); + if (close == null || prices.Count == 0) + { + return null; + } + + var onAStep = prices.Count(price => UnitStep(Math.Log10(price / (double)close.Value)) != null); + return onAStep * 2 >= prices.Count; + } + + /// + /// The thousandfold step a line's price sits on against the close, given as log10 of their + /// ratio: 0 for whole dollars, -1 for thousands, 1 for a VALUE typed a thousand times too + /// large. Null when it sits on none, within PriceMatchTolerance, which is a price that says + /// nothing about the unit: another security, a bond at par against a stock, or a slip. + /// Scaling those by the nearest step put Seagate's December 2025 quarter at $413 billion. + /// + internal static int? UnitStep(double offset) + { + var steps = (int)Math.Round(offset / 3); + return Math.Abs(steps) <= 1 && Math.Abs(offset - 3 * steps) <= PriceMatchTolerance ? steps : null; + } + + /// + /// Whether a CUSIP's issue number is an equity one: two digits, where debt uses letters. The + /// CUSIP is brought to nine characters first, since filers drop leading zeros and check digits. + /// + internal static bool IsEquityIssue(string cusip) + { + var nine = NormalizeCusip(cusip); + return nine != null && char.IsDigit(nine[6]) && char.IsDigit(nine[7]); + } + + /// + /// Resolves a CUSIP through the N-PORT ticker crosswalk. The ticker is resolved on the day + /// the funds reported it, since a ticker names a security only on a date: at a 2021 filing + /// META named a Roundhill ETF, not Facebook. The map file is checked first because + /// GenerateEquity never returns null for an unknown ticker. Whether the group it resolves is + /// kept is KeepsCrosswalkGroup's call, on that day's prices. + /// + internal SecurityIdentifier ResolveThroughTicker(string cusip) + { + if (!TickerCrosswalk.TryGetValue(cusip, out var entry)) + { + return null; + } + + if (!_crosswalkSecurities.TryGetValue(cusip, out var security)) + { + var mapFile = _mapFileProvider + .Get(new AuxiliaryDataKey(Market.USA, SecurityType.Equity)) + .ResolveMapFile(entry.Ticker, entry.Observed); + + // The map file has to carry the ticker on that very date. Barrick reached the + // crosswalk as ABX, its Toronto ticker, when ABX named no US security, and the + // resolver still returned the company that took ABX months later. + var tradedUnderIt = mapFile.Any() && string.Equals( + mapFile.GetMappedSymbol(entry.Observed, null), entry.Ticker, StringComparison.OrdinalIgnoreCase); + + _crosswalkSecurities[cusip] = security = tradedUnderIt + ? SecurityIdentifier.GenerateEquity(mapFile.FirstDate, mapFile.FirstTicker, Market.USA) + : null; + } + + return security; + } + + /// + /// Builds the US ISIN for a nine character CUSIP: the country code, the CUSIP, and the + /// check digit, which is arithmetic rather than a lookup. "037833100" gives "US0378331005". + /// + internal static string BuildUnitedStatesIsin(string cusip) + { + var body = "US" + cusip; + return body + ComputeIsinCheckDigit(body); + } + + /// + /// Computes an ISIN check digit: expand each letter to its ordinal (A is 10 through Z is 35), + /// then run Luhn over the resulting digits from the right. + /// + private static char ComputeIsinCheckDigit(string body) + { + var digits = new StringBuilder(body.Length * 2); + foreach (var character in body) + { + if (character >= '0' && character <= '9') + { + digits.Append(character); + } + else if (character >= 'A' && character <= 'Z') + { + digits.Append((character - 'A' + 10).ToStringInvariant()); + } + else + { + throw new FormatException( + $"SEC13FDownloader.ComputeIsinCheckDigit(): '{body}' carries '{character}', which is neither " + + "a digit nor an upper case letter"); + } + } + + var sum = 0; + var doubled = true; + for (var i = digits.Length - 1; i >= 0; i--) + { + var value = digits[i] - '0'; + if (doubled) + { + value *= 2; + if (value > 9) + { + value -= 9; + } + } + + sum += value; + doubled = !doubled; + } + + return (char)('0' + (10 - sum % 10) % 10); + } + + /// + /// The ticker a security traded under on a date, or null when its map file does not cover + /// it. No fallback to the last ticker: managers report dead CUSIPs for years, and by then the + /// ticker can belong to another company. + /// + internal string ResolveTicker(SecurityIdentifier security, DateTime tradingDate) + { + // Before its first row a map file answers with its first ticker, for a security that + // did not trade yet. + var mapFile = MapFileOf(security); + if (mapFile == null || tradingDate < mapFile.FirstDate) + { + return null; + } + + var ticker = mapFile.GetMappedSymbol(tradingDate, null); + return string.IsNullOrEmpty(ticker) ? null : ticker; + } + + /// The map file of a security, cached, or null when it has none. + private MapFile MapFileOf(SecurityIdentifier security) + { + if (!_mapFiles.TryGetValue(security, out var mapFile)) + { + _mapFiles[security] = mapFile = _mapFileProvider + .Get(AuxiliaryDataKey.Create(security)) + .ResolveMapFile(security.Symbol, security.Date); + } + + return mapFile; + } + + /// + /// Queues an increment for a security, to be staged when the archive is done, summing it into + /// whatever this archive already holds for the same quarter and release date. + /// + private void Queue(string security, string ticker, HoldingsRow row) + { + if (!_pendingSecurityRows.TryGetValue(security, out var rows)) + { + _pendingSecurityRows[security] = rows = new Dictionary<(DateTime, DateTime), (string, HoldingsRow)>(); + } + + var key = (row.PeriodEnd, row.Time); + rows[key] = rows.TryGetValue(key, out var queued) ? (queued.Ticker, Add(queued.Row, row)) : (ticker, row); + } + + /// Sums two increments reported for the same security, quarter and release date. + private static HoldingsRow Add(HoldingsRow left, HoldingsRow right) + { + var values = new decimal[ValueColumnCount]; + for (var i = 0; i < ValueColumnCount; i++) + { + values[i] = left.Values[i] + right.Values[i]; + } + + return new HoldingsRow + { + Time = left.Time, + PeriodEnd = left.PeriodEnd, + Values = values, + ConfidentialOmitted = left.ConfidentialOmitted || right.ConfidentialOmitted + }; + } + + /// + /// Appends the archive's increments to the staging file of their security; the merge happens + /// once, in the finalize pass. + /// + private void FlushPendingRows() + { + Directory.CreateDirectory(_stagingDirectory); + foreach (var (security, rows) in _pendingSecurityRows) + { + File.AppendAllLines( + StagingPath(security), + rows.Values + .OrderBy(entry => entry.Row.PeriodEnd) + .ThenBy(entry => entry.Row.Time) + .Select(entry => $"{entry.Ticker},{FormatRow(entry.Row)}")); + _stagedSecurities.Add(security); + } + + _pendingSecurityRows.Clear(); + } + + /// The staging file of a security. The identifier carries a space, so it is escaped. + private string StagingPath(string security) + { + return Path.Combine(_stagingDirectory, Uri.EscapeDataString(security) + ".csv"); + } + + /// + /// Turns the staged increments into the running total per quarter and writes the ticker + /// files. The total is kept per security, so a quarter still filing at a rename carries on in + /// the new ticker's file. An incremental run folds in the security's published rows from + /// every ticker it traded under, and keeps the rows of other securities in a shared file. + /// + private void FinalizeSecurityFiles() + { + if (_stagedSecurities.Count == 0) + { + Log.Trace("SEC13FDownloader.FinalizeSecurityFiles(): nothing was written"); + return; + } + + var incremental = _deploymentDate != null; + var tickerStaging = Path.Combine(_stagingDirectory, "tickers"); + Directory.CreateDirectory(tickerStaging); + + var shelfRows = 0L; + foreach (var security in _stagedSecurities) + { + var published = incremental + ? PublishedRowsOf(security) + : new List<(string Ticker, HoldingsRow Row)>(); + shelfRows += published.Count; + + var merged = MergeSecurity(published, ReadStagedRows(StagingPath(security))); + foreach (var ticker in merged.GroupBy(entry => entry.Ticker)) + { + File.AppendAllLines(Path.Combine(tickerStaging, $"{ticker.Key}.csv"), + ticker.Select(entry => FormatRow(entry.Row))); + } + } + + // A ticker file can hold the rows of more than one security, one after another as the + // ticker changed hands, so each is written once every security has been merged into it. + Directory.CreateDirectory(_destinationDirectory); + var files = Directory.GetFiles(tickerStaging, "*.csv"); + var keptRows = 0L; + + foreach (var file in files) + { + var ticker = Path.GetFileNameWithoutExtension(file); + var rows = ReadRows(file).ToList(); + + if (incremental) + { + var others = OtherSecuritiesRows(ticker); + keptRows += others.Count; + rows.AddRange(others); + } + + // Time order: LEAN's SubscriptionDataReader silently drops a point whose timestamp + // moves backwards. The quarters of one release are written oldest first, inside the + // single line that carries them. + File.WriteAllLines( + Path.Combine(_destinationDirectory, $"{ticker}.csv"), + rows.OrderBy(row => row.Time).ThenBy(row => row.PeriodEnd) + .GroupBy(row => row.Time) + .Select(release => FormatPoint(release.Key, release.ToList()))); + } + + // The shelf row count is what tells a merge from a silent rebuild, so it is logged. + Log.Trace($"SEC13FDownloader.FinalizeSecurityFiles(): {_stagedSecurities.Count} securities merged into " + + $"{files.Length} ticker files, {shelfRows} of their rows read from {_processedDataDirectory}, " + + $"{keptRows} rows of other securities kept unchanged"); + + Directory.Delete(_stagingDirectory, recursive: true); + } + + /// + /// The published rows of a security, from every ticker it ever traded under. A ticker file + /// keeps only the rows written while the security owned the ticker, which is the same test + /// EmitHoldings applies before writing one. + /// + private List<(string Ticker, HoldingsRow Row)> PublishedRowsOf(string security) + { + var rows = new List<(string Ticker, HoldingsRow Row)>(); + var mapFile = MapFileOf(SecurityIdentifier.Parse(security)); + if (mapFile == null) + { + return rows; + } + + foreach (var ticker in mapFile.Select(row => row.MappedSymbol.ToLowerInvariant()).Distinct()) + { + var path = Path.Combine(_processedDataDirectory, $"{ticker}.csv"); + if (!IsFileNameSafe(ticker) || !File.Exists(path)) + { + continue; + } + + rows.AddRange(ReadPublishedRows(path) + .Where(row => TickerOwner(ticker, row.Time.Date) == security) + .Select(row => (ticker, row))); + } + + // Only this security's dates are held, and the next one asks about others. + _tickerOwners.Clear(); + return rows; + } + + /// + /// The published rows of a ticker that belong to a security this run did not touch, which + /// stay as they are. The rows of a touched security come back through its own merge. + /// + private List OtherSecuritiesRows(string ticker) + { + var path = Path.Combine(_processedDataDirectory, $"{ticker}.csv"); + if (!File.Exists(path)) + { + return new List(); + } + + var rows = ReadPublishedRows(path) + .Where(row => + { + var owner = TickerOwner(ticker, row.Time.Date); + return owner == null || !_stagedSecurities.Contains(owner); + }) + .ToList(); + + _tickerOwners.Clear(); + return rows; + } + + /// Reads a staging file back: the ticker, then the increment in the published layout. + private static IEnumerable<(string Ticker, HoldingsRow Row)> ReadStagedRows(string path) + { + foreach (var line in File.ReadLines(path)) + { + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var comma = line.IndexOf(','); + yield return (line.Substring(0, comma), ParseRow(line.Substring(comma + 1))); + } + } + + /// + /// Rebuilds the universe from the per-security files and forward-fills it across business + /// days, so each file is the whole cross-section on its date rather than who filed that day. + /// A row only replaces one of an equal or older quarter, since amendments arrive years late + /// and must not overwrite newer figures. A quarter leaves once it is older than + /// OldestLivePeriod, and a security after its delisting date. + /// + internal void BuildUniverseFiles() + { + // Release date to the rows public that day. An incremental run also reads the published + // files, this run's copy of a ticker winning; a full run replaces them and reads only its own. + var events = new SortedDictionary>(); + var securities = new HashSet(StringComparer.OrdinalIgnoreCase); + var unmapped = 0; + var malformed = 0; + + var directories = _deploymentDate == null + ? new[] { _destinationDirectory } + : new[] { _destinationDirectory, _processedDataDirectory }; + + foreach (var directory in directories) + { + if (!Directory.Exists(directory)) + { + continue; + } + + foreach (var file in Directory.EnumerateFiles(directory, "*.csv")) + { + var ticker = Path.GetFileNameWithoutExtension(file).ToUpperInvariant(); + if (!securities.Add(ticker)) + { + continue; + } + + foreach (var line in File.ReadAllLines(file)) + { + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var csv = line.Split(','); + if (csv.Length < PointHeaderColumns + GroupColumnCount || + !int.TryParse(csv[1], NumberStyles.Integer, CultureInfo.InvariantCulture, + out var quarters) || + quarters <= 0 || + !DateTime.TryParseExact(csv[0], ReleaseFormat, CultureInfo.InvariantCulture, + DateTimeStyles.None, out var release)) + { + malformed++; + continue; + } + + var releaseDate = release.Date; + + // The owner of the ticker on the release date. The map file is checked first + // because GenerateEquity never returns null for an unknown ticker. + var mapFile = _mapFileProvider + .Get(new AuxiliaryDataKey(Market.USA, SecurityType.Equity)) + .ResolveMapFile(ticker, releaseDate); + if (!mapFile.Any()) + { + unmapped++; + continue; + } + + var security = SecurityIdentifier + .GenerateEquity(mapFile.FirstDate, mapFile.FirstTicker, Market.USA); + + if (!events.TryGetValue(releaseDate, out var rows)) + { + events[releaseDate] = rows = new List(); + } + + var width = (csv.Length - PointHeaderColumns) / quarters; + for (var quarter = 0; quarter < quarters; quarter++) + { + var offset = PointHeaderColumns + quarter * width; + if (!DateTime.TryParseExact(csv[offset], PeriodFormat, CultureInfo.InvariantCulture, + DateTimeStyles.None, out var periodEnd)) + { + malformed++; + continue; + } + + // The published columns are carried unchanged, so the two views cannot disagree. + rows.Add(new UniverseRow( + security.ToString(), + periodEnd, + ParseDecimal(csv[offset + 1]), + mapFile.DelistingDate, + ticker, + string.Join(",", csv.Skip(offset).Take(width)))); + } + } + } + } + + if (unmapped > 0 || malformed > 0) + { + Log.Trace($"SEC13FDownloader.BuildUniverseFiles(): {unmapped} rows skipped, the map files know no " + + $"security under their ticker on the release date; {malformed} rows unreadable"); + } + + if (events.Count == 0) + { + Log.Trace("SEC13FDownloader.BuildUniverseFiles(): nothing to forward-fill"); + return; + } + + var universeDirectory = Path.Combine(_destinationDirectory, "universe"); + Directory.CreateDirectory(universeDirectory); + + // Keyed by SecurityIdentifier, the identity that survives a rename. Two quarters are + // carried per security: a new quarter fills in over about fifty days, so publishing only + // it would rank names by who filed first, and publishing only the finished one would + // hide the fresh data. The older leaves once the newer overtakes it in Holders. + var current = new Dictionary>(StringComparer.Ordinal); + var written = 0; + var superseded = 0; + var ignoredAsStale = 0; + var expiredQuarters = 0; + var delistedSecurities = 0; + var rowsWritten = 0L; + + for (var day = events.Keys.First(); day <= events.Keys.Last(); day = day.AddDays(1)) + { + var oldestLive = OldestLivePeriod(day); + + if (events.TryGetValue(day, out var todays)) + { + foreach (var row in todays) + { + current.TryGetValue(row.SecurityIdentifier, out var periods); + + // A late row for a retired or expired quarter is not resurrected. + if (row.PeriodEnd < oldestLive || periods?.Count > 0 && row.PeriodEnd < periods.Keys[0]) + { + ignoredAsStale++; + continue; + } + + if (periods == null) + { + current[row.SecurityIdentifier] = periods = new SortedList(); + } + + if (periods.ContainsKey(row.PeriodEnd)) + { + superseded++; + } + + periods[row.PeriodEnd] = row; + RetireOvertakenQuarters(periods); + } + } + + // Every day, weekends included, so the shelf life does not depend on which weekday + // a deadline falls on. + foreach (var security in current.Keys.ToList()) + { + var periods = current[security]; + while (periods.Count > 0 && periods.Keys[0] < oldestLive) + { + periods.RemoveAt(0); + expiredQuarters++; + } + + if (periods.Count == 0) + { + current.Remove(security); + } + else if (periods.Values[0].DelistingDate < day) + { + current.Remove(security); + delistedSecurities++; + } + } + + if (day.DayOfWeek == DayOfWeek.Saturday || day.DayOfWeek == DayOfWeek.Sunday || current.Count == 0) + { + continue; + } + + // One line per security, its quarters oldest first, securities in identifier + // order. LEAN hands the selection function a single record per security. + var rows = current + .OrderBy(security => security.Key, StringComparer.Ordinal) + .Select(security => FormatUniverseLine(security.Key, security.Value.Values)) + .ToList(); + + File.WriteAllLines(Path.Combine(universeDirectory, $"{day.ToStringInvariant("yyyyMMdd")}.csv"), rows); + written++; + rowsWritten += rows.Count; + } + + Log.Trace($"SEC13FDownloader.BuildUniverseFiles(): forward-filled {written} business days " + + $"from {events.Keys.First():yyyy-MM-dd} to {events.Keys.Last():yyyy-MM-dd}, " + + $"{current.Count} securities and " + + $"{current.Values.Sum(periods => periods.Count)} rows in the last file, " + + $"{rowsWritten} lines written, {superseded} restated, " + + $"{ignoredAsStale} late rows ignored as stale, {expiredQuarters} quarters expired, " + + $"{delistedSecurities} securities delisted"); + } + + /// + /// Keeps the newest quarter, and the one before it only while it still holds more managers. + /// + private static void RetireOvertakenQuarters(SortedList periods) + { + while (periods.Count > 1) + { + var newest = periods.Values[periods.Count - 1].Holders; + var previous = periods.Values[periods.Count - 2].Holders; + + if (periods.Count > 2 || previous <= newest) + { + periods.RemoveAt(0); + continue; + } + + break; + } + } + + /// + /// The oldest quarter still live on a day: the one before the newest finished quarter stays + /// live until that quarter's 45 day filing deadline passes. + /// + internal static DateTime OldestLivePeriod(DateTime day) + { + var latest = QuarterEndBefore(day.Date); + return day.Date > latest.AddDays(FilingDeadlineDays) ? latest : QuarterEndBefore(latest); + } + + /// The last quarter end strictly before a day. + private static DateTime QuarterEndBefore(DateTime day) + { + return new DateTime(day.Year, (day.Month - 1) / 3 * 3 + 1, 1).AddDays(-1); + } + + /// + /// One universe line: the identifier, the ticker the security traded under when its newest + /// quarter was published, and that quarter and the one before it, oldest first. + /// + private static string FormatUniverseLine(string security, IList quarters) + { + var line = new StringBuilder() + .Append(security).Append(',') + .Append(quarters[quarters.Count - 1].Ticker).Append(',') + .Append(quarters.Count.ToStringInvariant()); + + foreach (var quarter in quarters) + { + line.Append(',').Append(quarter.Group); + } + + return line.ToString(); + } + + /// One quarter of a universe line, with the fields the forward-fill reads. + private readonly record struct UniverseRow( + string SecurityIdentifier, DateTime PeriodEnd, decimal Holders, DateTime DelistingDate, + string Ticker, string Group); + + /// One row of a per-security file, either an increment or the running total. + internal sealed class HoldingsRow + { + public DateTime Time { get; init; } + public DateTime PeriodEnd { get; init; } + public decimal[] Values { get; init; } + public bool ConfidentialOmitted { get; init; } + } + + /// + /// Folds a security's published rows and this run's increments into the running total per + /// quarter, each row tagged with its ticker file, in time order. The published rows are + /// differenced back to increments first, which is exact in decimal, and across every ticker + /// the security traded under, which is what carries a quarter across a rename. + /// + internal static List<(string Ticker, HoldingsRow Row)> MergeSecurity( + IEnumerable<(string Ticker, HoldingsRow Row)> published, + IEnumerable<(string Ticker, HoldingsRow Row)> increments) + { + var merged = new Dictionary<(DateTime PeriodEnd, DateTime Time), (string Ticker, HoldingsRow Row)>(); + + var shelf = published.ToList(); + var shelfTickers = new Dictionary<(DateTime PeriodEnd, DateTime Time), string>(); + foreach (var (ticker, row) in shelf) + { + shelfTickers[(row.PeriodEnd, row.Time)] = ticker; + } + + foreach (var row in Difference(shelf.Select(entry => entry.Row))) + { + var key = (row.PeriodEnd, row.Time); + merged[key] = (shelfTickers[key], row); + } + + // Within this run, two increments for one key are two different filings and are summed. + var fresh = new Dictionary<(DateTime PeriodEnd, DateTime Time), (string Ticker, HoldingsRow Row)>(); + foreach (var (ticker, row) in increments) + { + var key = (row.PeriodEnd, row.Time); + fresh[key] = fresh.TryGetValue(key, out var held) ? (held.Ticker, Add(held.Row, row)) : (ticker, row); + } + + // Against the published history this run wins: it read the archive again, so its + // reading of a quarter and release date is the current one, and adding would double it. + foreach (var (key, entry) in fresh) + { + merged[key] = entry; + } + + return Accumulate(merged.Values.Select(entry => entry.Row)) + .Select(row => (Ticker: merged[(row.PeriodEnd, row.Time)].Ticker, Row: row)) + .OrderBy(entry => entry.Row.Time) + .ThenBy(entry => entry.Row.PeriodEnd) + .ToList(); + } + + /// Reads a published per-security file, unpacking every release into its quarters. + private static IEnumerable ReadPublishedRows(string path) + { + return File.ReadAllLines(path) + .Where(line => !string.IsNullOrWhiteSpace(line)) + .SelectMany(ParsePoint); + } + + /// Unpacks one published line into one row per quarter it restates. + private static IEnumerable ParsePoint(string line) + { + var csv = line.Split(','); + var time = DateTime.ParseExact(csv[0], ReleaseFormat, CultureInfo.InvariantCulture); + var quarters = int.Parse(csv[1], NumberStyles.Integer, CultureInfo.InvariantCulture); + var width = (csv.Length - PointHeaderColumns) / quarters; + + for (var quarter = 0; quarter < quarters; quarter++) + { + yield return ParseGroup(time, csv, PointHeaderColumns + quarter * width); + } + } + + /// Reads a staging file, skipping blank lines. + private static IEnumerable ReadRows(string path) + { + return File.ReadAllLines(path).Where(line => !string.IsNullOrWhiteSpace(line)).Select(ParseRow); + } + + /// + /// Turns the running total a published file holds back into the increments it was built + /// from, quarter by quarter and in release order. + /// + internal static IEnumerable Difference(IEnumerable rows) + { + foreach (var period in rows.GroupBy(row => row.PeriodEnd)) + { + var previous = new decimal[ValueColumnCount]; + foreach (var row in period.OrderBy(row => row.Time)) + { + var values = new decimal[ValueColumnCount]; + for (var i = 0; i < ValueColumnCount; i++) + { + values[i] = row.Values[i] - previous[i]; + previous[i] = row.Values[i]; + } + + // The flag is carried rather than differenced: it is OR'd forward on the way + // back, and OR is idempotent, so this round-trips whatever it was set from. + yield return new HoldingsRow + { + Time = row.Time, + PeriodEnd = row.PeriodEnd, + Values = values, + ConfidentialOmitted = row.ConfidentialOmitted + }; + } + } + } + + /// + /// Running-sums the increments within each quarter: every published row states what had been + /// reported for its quarter as of its own release date. + /// + internal static List Accumulate(IEnumerable increments) + { + var cumulative = new List(); + + foreach (var period in increments.GroupBy(row => row.PeriodEnd).OrderBy(period => period.Key)) + { + var running = new decimal[ValueColumnCount]; + var confidential = false; + + foreach (var row in period.OrderBy(row => row.Time)) + { + var values = new decimal[ValueColumnCount]; + for (var i = 0; i < ValueColumnCount; i++) + { + running[i] += row.Values[i]; + values[i] = running[i]; + } + + // Once a contributing filing withheld positions the quarter's total stays + // incomplete, so the flag only ever goes up within a quarter. + confidential |= row.ConfidentialOmitted; + + cumulative.Add(new HoldingsRow + { + Time = row.Time, + PeriodEnd = row.PeriodEnd, + Values = values, + ConfidentialOmitted = confidential + }); + } + } + + return cumulative; + } + + /// Parses one staged row. + private static HoldingsRow ParseRow(string line) + { + var csv = line.Split(','); + return ParseGroup(DateTime.ParseExact(csv[0], ReleaseFormat, CultureInfo.InvariantCulture), csv, 1); + } + + /// Reads one quarter: the period, the eight measures and the confidential flag. + private static HoldingsRow ParseGroup(DateTime time, string[] csv, int offset) + { + var values = new decimal[ValueColumnCount]; + for (var i = 0; i < ValueColumnCount; i++) + { + values[i] = ParseDecimal(csv[offset + 1 + i]); + } + + return new HoldingsRow + { + Time = time, + PeriodEnd = DateTime.ParseExact(csv[offset], PeriodFormat, CultureInfo.InvariantCulture), + Values = values, + ConfidentialOmitted = csv[offset + 1 + ValueColumnCount] == "1" + }; + } + + /// Formats one staged row: the release stamp and the quarter behind it. + private static string FormatRow(HoldingsRow row) + { + return $"{row.Time.ToStringInvariant(ReleaseFormat)},{FormatGroup(row)}"; + } + + /// Formats one release, in the layout SEC13F parses. + private static string FormatPoint(DateTime time, IList quarters) + { + var line = new StringBuilder() + .Append(time.ToStringInvariant(ReleaseFormat)).Append(',') + .Append(quarters.Count.ToStringInvariant()); + + foreach (var row in quarters) + { + line.Append(',').Append(FormatGroup(row)); + } + + return line.ToString(); + } + + /// Formats one quarter of a release, in the layout SEC13FHolding parses. + private static string FormatGroup(HoldingsRow row) + { + var group = new StringBuilder().Append(row.PeriodEnd.ToStringInvariant(PeriodFormat)); + + foreach (var value in row.Values) + { + group.Append(',').Append(FormatValue(value)); + } + + return group.Append(',').Append(row.ConfidentialOmitted ? '1' : '0').ToString(); + } + + /// Reports how much of the dataset made it through identity resolution. + private void LogResolutionSummary() + { + var totalValue = _resolvedValue + _unresolvedValue; + var coverage = totalValue == 0m ? 0m : 100m * _resolvedValue / totalValue; + + Log.Trace($"SEC13FDownloader.LogResolutionSummary(): {_countedFilers.Count} counted filers " + + $"over {_securityIndexes.Count} securities, " + + $"working set {GC.GetTotalMemory(false) / (1024 * 1024)} MB"); + + Log.Trace($"SEC13FDownloader.LogResolutionSummary(): {_resolvedByCusip} distinct CUSIPs resolved by CUSIP, " + + $"{_resolvedByIsin} by constructed ISIN, {_resolvedByTicker} by the N-PORT ticker crosswalk, " + + $"{_unresolvedCusips.Count} unresolved, {_malformedCusips} malformed"); + Log.Trace($"SEC13FDownloader.LogResolutionSummary(): {_unresolvedGroups} groups dropped, " + + $"{_conflictingTickers} of them, {(totalValue == 0m ? 0m : 100m * _conflictingValue / totalValue).ToStringInvariant("F2")}% " + + "of reported value, because the ticker belonged to another security that day, " + + $"{coverage.ToStringInvariant("F1")}% of reported value covered, " + + $"{_unchangedGroups} resolved groups added nothing new and wrote no row"); + Log.Trace($"SEC13FDownloader.LogResolutionSummary(): {_mismatchedGroups} crosswalk groups dropped because " + + $"their prices were another security's, {_debtCusipsRejected} groups of CUSIPs with letters in the issue " + + "number dropped for want of a price matching the close, " + + $"{(totalValue == 0m ? 0m : 100m * _mismatchedValue / totalValue).ToStringInvariant("F2")}% of reported value together"); + + if (_unresolvedCusips.Count > 0) + { + Log.Trace($"SEC13FDownloader.LogResolutionSummary(): unresolved sample: " + + $"{string.Join(", ", _unresolvedCusips.Take(25))}"); + } + + if (_conflictSample.Count > 0) + { + Log.Trace($"SEC13FDownloader.LogResolutionSummary(): ticker conflict sample: {string.Join("; ", _conflictSample)}"); + } + } + + /// One table of an archive, where a short row means the layout moved and throws. + private static IEnumerable<(Dictionary Columns, string[] Fields)> ReadTable( + ZipArchive zip, Archive archive, string table, params string[] required) + { + return SEC13FFiles.ReadTable(zip, archive.Name, table, skipShortRows: false, required); + } + + /// + /// Parses a date as the SEC writes it in these tables, "31-MAR-2026", with the ISO and US + /// shapes accepted too because the older archives are not perfectly consistent. + /// + private static DateTime ParseSecDate(string value, Archive archive, string column) + { + // Some rows carry a time of day the tables do not otherwise use. + var date = value.Trim().Split(' ')[0]; + if (DateTime.TryParseExact(date, SecDateFormats, CultureInfo.InvariantCulture, + DateTimeStyles.None, out var parsed)) + { + return parsed; + } + + throw new FormatException( + $"SEC13FDownloader.ParseSecDate(): {archive.Name} {column} '{value}' matches none of " + + $"{string.Join(", ", SecDateFormats)}"); + } + + /// Parses a reported amount, treating a blank as zero. + private static decimal ParseDecimal(string value) + { + var trimmed = value.Trim(); + if (trimmed.Length == 0) + { + return 0m; + } + + if (!decimal.TryParse(trimmed, NumberStyles.Any, CultureInfo.InvariantCulture, out var parsed)) + { + throw new FormatException($"SEC13FDownloader.ParseDecimal(): '{value}' is not a number"); + } + + return parsed; + } + + /// True for the "Y" the SEC writes in its flag columns. + private static bool IsYes(string value) + { + return value.Trim().Equals("Y", StringComparison.OrdinalIgnoreCase); + } + + /// Formats an amount: whole numbers without a decimal point, everything else invariant. + private static string FormatValue(decimal value) + { + return value == Math.Truncate(value) + ? ((long)value).ToStringInvariant() + : value.ToStringInvariant(); + } + + /// + /// True when the ticker can name a file LEAN will ask for. Skips path separators and invalid + /// file name characters, and the space and '|' a Symbol cannot hold: the map files carry + /// "ua.c " for Under Armour's class C, and its file could never be read. + /// + internal static bool IsFileNameSafe(string ticker) + { + return ticker.IndexOf('/') < 0 + && ticker.IndexOf('\\') < 0 + && ticker.IndexOf('|') < 0 + && !ticker.Any(char.IsWhiteSpace) + && ticker.IndexOfAny(Path.GetInvalidFileNameChars()) < 0; + } + + /// + /// Sets the User-Agent the SEC asks automated readers for, from the same config keys the + /// reports dataset reads, on the first request: a run that never reaches the network, like + /// the unit tests, does not need them. + /// + private void RequireUserAgent() + { + if (_userAgentSet) + { + return; + } + + var companyName = Config.Get("sec-user-agent-company-name"); + var companyEmail = Config.Get("sec-user-agent-company-email"); + if (string.IsNullOrEmpty(companyName) || string.IsNullOrEmpty(companyEmail)) + { + throw new ArgumentException("The SEC requires a company name and email to download data using " + + "automation. Set `sec-user-agent-company-name` and `sec-user-agent-company-email` in the config."); + } + + _client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", string.Join(" ", companyName, companyEmail)); + _userAgentSet = true; + } + + /// GETs a URL as text, with retry and backoff. + private string GetWithRetry(string url) + { + return WithRetry(url, () => + { + using var response = _client.GetAsync(url).GetAwaiter().GetResult(); + response.EnsureSuccessStatusCode(); + return response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + }); + } + + /// + /// The names in one of EDGAR's directory listings. Read once a run, so a day published after + /// the run read its quarter is left to the next run. Fails like any other GET, a block included. + /// + private ISet ListEdgarDirectory(string url) + { + if (!_edgarListings.TryGetValue(url, out var names)) + { + names = SEC13FEdgarDay.ListingNames(GetWithRetry(url + "index.json")); + _edgarListings[url] = names; + } + + return names; + } + + /// + /// Sends a request through the rate gate, retrying whatever IsWorthRetrying accepts with a + /// backoff that doubles up to two minutes, about four minutes in all: EDGAR served a new + /// filer's listed filing as a 404 for over a minute. The last failure is the one that surfaces. + /// + private T WithRetry(string url, Func request) + { + RequireUserAgent(); + for (var attempt = 1; ; attempt++) + { + try + { + _rateGate.WaitToProceed(); + return request(); + } + catch (Exception err) when (attempt < MaxRetries && IsWorthRetrying(err)) + { + Log.Trace($"SEC13FDownloader.WithRetry(): {url} retry {attempt}/{MaxRetries} after: {err.Message}"); + Thread.Sleep(TimeSpan.FromSeconds(Math.Min(120, Math.Pow(2, attempt)))); + } + } + } + + /// + /// True when a failed request is worth asking again: transport errors, server errors, + /// throttling and a 404. Every file requested is one the SEC lists, so a 404 is its own + /// hiccup: the first filings of two new filers, in the 24 and 27 July 2026 indexes, answered + /// 404 for up to a minute and 200 afterwards. A 403 is a block, which outlasts any backoff. + /// + internal static bool IsWorthRetrying(Exception error) + { + var status = (error as HttpRequestException)?.StatusCode; + + return status == null + || (int)status >= 500 + || status == HttpStatusCode.NotFound + || status == HttpStatusCode.TooManyRequests + || status == HttpStatusCode.RequestTimeout; + } + + /// Disposes unmanaged resources. + public void Dispose() + { + _client.DisposeSafely(); + _rateGate.DisposeSafely(); + + // A run that failed half way leaves its staging behind, and nothing else ever reads it. + try + { + if (Directory.Exists(_stagingDirectory)) + { + Directory.Delete(_stagingDirectory, recursive: true); + } + } + catch (IOException err) + { + Log.Error(err, $"SEC13FDownloader.Dispose(): could not delete {_stagingDirectory}"); + } + + GC.SuppressFinalize(this); + } + } +} diff --git a/DataProcessing/SEC13FEdgarDay.cs b/DataProcessing/SEC13FEdgarDay.cs new file mode 100644 index 0000000..e27d8a8 --- /dev/null +++ b/DataProcessing/SEC13FEdgarDay.cs @@ -0,0 +1,317 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Xml.Linq; +using Newtonsoft.Json; +using QuantConnect.DataSource; +using QuantConnect.Logging; + +namespace QuantConnect.DataProcessing +{ + /// + /// One day of Form 13F filings read straight from EDGAR and written as an archive with the + /// tables the SEC data sets carry, so the processor reads it like any other window. + /// + /// The data sets come out in three month batches a few days after each window closes. A daily + /// job waiting for them publishes a filing up to three months after it was public, while a + /// backtest stamped at the filing date sees it the day it was filed. EDGAR lists each day's + /// filings in its daily index that night, and their information tables carry the lines the data + /// sets do: 48 of 48 filings compared line by line, and the same accession numbers on six sample + /// days, so reading EDGAR is what lets the live job publish on the date the history uses. + /// + internal static class SEC13FEdgarDay + { + private const string DailyIndexRootUrl = "https://www.sec.gov/Archives/edgar/daily-index/"; + private const string ArchivesBaseUrl = "https://www.sec.gov/Archives/"; + + private const string PrimaryDocumentTypePrefix = "13F-HR"; + private const string InformationTableDocumentType = "INFORMATION TABLE"; + + private static readonly Regex IndexLine = new( + @"^(?
13F-HR(?:/A)?)\s+.+?\s+(?\d+)\s+(?\d{8})\s+(?edgar/data/\S+\.txt)\s*$", + RegexOptions.Compiled); + + private static readonly Regex DocumentBlock = new(@"(?.*?)", + RegexOptions.Compiled | RegexOptions.Singleline); + + private static readonly Regex DocumentType = new(@"(?[^\r\n<]+)", RegexOptions.Compiled); + + private static readonly Regex XmlBlock = new(@"(?.*?)", + RegexOptions.Compiled | RegexOptions.Singleline); + + /// One holdings filing listed in a daily index. + internal readonly record struct IndexEntry(string FormType, int Cik, DateTime Filed, string Path) + { + /// The accession number, which is the name of the submission file. + public string Accession => System.IO.Path.GetFileNameWithoutExtension(Path); + } + + /// What the archive's tables carry for one filing. + internal sealed class Filing + { + public string Accession { get; init; } + public string SubmissionType { get; init; } + public int Cik { get; init; } + public DateTime Filed { get; init; } + public DateTime Period { get; init; } + public bool ConfidentialOmitted { get; init; } + + /// INFOTABLE rows: CUSIP, VALUE, SSHPRNAMT, SSHPRNAMTTYPE, PUTCALL, VOTING_AUTH_SOLE, VOTING_AUTH_SHARED. + public List Lines { get; } = new(); + } + + /// The name a day's archive is cached and logged under. + public static string ArchiveName(DateTime day) => $"edgar-{day.ToString(DateFormat.EightCharacter, CultureInfo.InvariantCulture)}_form13f.zip"; + + /// + /// Writes the day's archive into and returns its path, or null + /// when EDGAR has not published an index for the day, which is every weekend and holiday and a + /// day whose index is late. returns the names in one of EDGAR's + /// directory listings and a file; both throw on any failure. + /// + public static string Build(DateTime day, string directory, Func> listDirectory, + Func getText) + { + var path = System.IO.Path.Combine(directory, ArchiveName(day)); + if (File.Exists(path)) + { + return path; + } + + if (!IsIndexPublished(day, listDirectory)) + { + return null; + } + + var entries = ParseIndex(getText(IndexUrl(day))); + var filings = new List(entries.Count); + foreach (var entry in entries) + { + filings.Add(ParseFiling(entry, getText(ArchivesBaseUrl + entry.Path))); + } + + Directory.CreateDirectory(directory); + SEC13FFiles.WriteThenMove(path, stream => WriteArchive(stream, filings)); + + Log.Trace($"SEC13FEdgarDay.Build(): {day:yyyy-MM-dd}: {filings.Count} holdings filings, " + + $"{filings.Sum(filing => filing.Lines.Count)} information table lines"); + return path; + } + + /// The daily form index, which lists a day's filings by form type. + internal static string IndexUrl(DateTime day) + { + return $"{QuarterUrl(day)}{IndexFileName(day)}"; + } + + /// + /// Whether EDGAR lists the day's form index. The answer comes from its directory listings, + /// never from a failed request: EDGAR answers 403 both for an index that does not exist and + /// for a reader it has blocked, and a block taken for "no index" dropped the day for good. Each + /// folder is looked up in its parent's listing first, from the root, which always exists: a + /// year or quarter EDGAR has not created yet answers 403 as well. + /// + internal static bool IsIndexPublished(DateTime day, Func> listDirectory) + { + var year = day.Year.ToString(CultureInfo.InvariantCulture); + return listDirectory(DailyIndexRootUrl).Contains(year) + && listDirectory($"{DailyIndexRootUrl}{year}/").Contains(QuarterName(day)) + && listDirectory(QuarterUrl(day)).Contains(IndexFileName(day)); + } + + /// The names in one of EDGAR's index.json directory listings. + internal static HashSet ListingNames(string json) + { + var listing = JsonConvert.DeserializeObject(json)?.Directory + ?? throw new InvalidDataException("SEC13FEdgarDay.ListingNames(): not an EDGAR directory listing"); + + return (listing.Items ?? new List()).Select(item => item.Name).ToHashSet(StringComparer.Ordinal); + } + + private static string QuarterName(DateTime day) => $"QTR{(day.Month - 1) / 3 + 1}"; + + private static string QuarterUrl(DateTime day) => + $"{DailyIndexRootUrl}{day.Year.ToString(CultureInfo.InvariantCulture)}/{QuarterName(day)}/"; + + private static string IndexFileName(DateTime day) => + $"form.{day.ToString(DateFormat.EightCharacter, CultureInfo.InvariantCulture)}.idx"; + + /// + /// The holdings reports and their amendments listed in a daily index. Notices are left out: + /// they carry no information table, and the processor skips them in the data sets too. + /// + internal static List ParseIndex(string text) + { + var entries = new List(); + foreach (var line in text.Split('\n')) + { + var match = IndexLine.Match(line.TrimEnd('\r')); + if (!match.Success) + { + continue; + } + + entries.Add(new IndexEntry( + match.Groups["form"].Value, + int.Parse(match.Groups["cik"].Value, CultureInfo.InvariantCulture), + DateTime.ParseExact(match.Groups["date"].Value, DateFormat.EightCharacter, CultureInfo.InvariantCulture), + match.Groups["file"].Value)); + } + + return entries; + } + + /// + /// Reads one full submission file: the period and confidential treatment flag from the + /// primary document, and every line of its information tables. The form type, filer and + /// filing date come from the index, as they do in the data sets. + /// + internal static Filing ParseFiling(IndexEntry entry, string text) + { + XElement primary = null; + var tables = new List(); + + foreach (Match document in DocumentBlock.Matches(text)) + { + var body = document.Groups["body"].Value; + var type = DocumentType.Match(body).Groups["type"].Value.Trim(); + var xml = XmlBlock.Match(body); + if (!xml.Success) + { + continue; + } + + if (type.StartsWith(PrimaryDocumentTypePrefix, StringComparison.OrdinalIgnoreCase)) + { + primary = Parse(xml.Groups["xml"].Value, entry); + } + else if (type.Equals(InformationTableDocumentType, StringComparison.OrdinalIgnoreCase)) + { + tables.Add(Parse(xml.Groups["xml"].Value, entry)); + } + } + + if (primary == null) + { + throw new InvalidDataException($"SEC13FEdgarDay.ParseFiling(): {entry.Path} carries no primary document"); + } + + var period = Value(primary, "periodOfReport") ?? Value(primary, "reportCalendarOrQuarter") + ?? throw new InvalidDataException($"SEC13FEdgarDay.ParseFiling(): {entry.Path} carries no period of report"); + + var filing = new Filing + { + Accession = entry.Accession, + SubmissionType = entry.FormType, + Cik = entry.Cik, + Filed = entry.Filed, + Period = DateTime.ParseExact(period, "MM-dd-yyyy", CultureInfo.InvariantCulture), + ConfidentialOmitted = IsTrue(Value(primary, "isConfidentialOmitted")) + }; + + foreach (var line in tables.SelectMany(table => Elements(table, "infoTable"))) + { + filing.Lines.Add(new[] + { + Value(line, "cusip"), + Value(line, "value"), + Value(line, "sshPrnamt"), + Value(line, "sshPrnamtType"), + Value(line, "putCall"), + Value(line, "Sole"), + Value(line, "Shared") + }); + } + + return filing; + } + + /// Writes the three tables the processor reads, in the data sets' layout. + internal static void WriteArchive(Stream stream, IEnumerable filings) + { + var submissions = new StringBuilder("ACCESSION_NUMBER\tFILING_DATE\tSUBMISSIONTYPE\tCIK\tPERIODOFREPORT\n"); + var summaries = new StringBuilder("ACCESSION_NUMBER\tISCONFIDENTIALOMITTED\n"); + var lines = new StringBuilder("ACCESSION_NUMBER\tCUSIP\tVALUE\tSSHPRNAMT\tSSHPRNAMTTYPE\tPUTCALL\tVOTING_AUTH_SOLE\tVOTING_AUTH_SHARED\n"); + + foreach (var filing in filings) + { + submissions.Append(string.Join('\t', filing.Accession, + filing.Filed.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), filing.SubmissionType, + filing.Cik.ToString(CultureInfo.InvariantCulture), filing.Period.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture))).Append('\n'); + summaries.Append(filing.Accession).Append('\t').Append(filing.ConfidentialOmitted ? "Y" : "N").Append('\n'); + + foreach (var line in filing.Lines) + { + lines.Append(filing.Accession); + foreach (var field in line) + { + lines.Append('\t').Append(Clean(field)); + } + + lines.Append('\n'); + } + } + + using var zip = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true); + foreach (var (name, content) in new[] { ("SUBMISSION.tsv", submissions), ("SUMMARYPAGE.tsv", summaries), ("INFOTABLE.tsv", lines) }) + { + using var writer = new StreamWriter(zip.CreateEntry(name, CompressionLevel.Optimal).Open()); + writer.Write(content.ToString()); + } + } + + private static XElement Parse(string xml, IndexEntry entry) + { + try + { + return XDocument.Parse(xml.Trim()).Root; + } + catch (Exception err) + { + throw new InvalidDataException($"SEC13FEdgarDay.Parse(): {entry.Path} carries malformed XML: {err.Message}", err); + } + } + + /// Elements by local name, whatever namespace and case the filer's software used. + private static IEnumerable Elements(XElement root, string name) + { + return root.DescendantsAndSelf().Where(element => element.Name.LocalName.Equals(name, StringComparison.OrdinalIgnoreCase)); + } + + private static string Value(XElement root, string name) + { + return Elements(root, name).FirstOrDefault()?.Value.Trim(); + } + + private static bool IsTrue(string value) + { + return value != null && (value.Equals("true", StringComparison.OrdinalIgnoreCase) || value.Equals("Y", StringComparison.OrdinalIgnoreCase)); + } + + /// A field as a table cell: tabs and line breaks would split the row. + private static string Clean(string value) + { + return value == null ? string.Empty : Regex.Replace(value, @"\s+", " ").Trim(); + } + } +} diff --git a/DataProcessing/SEC13FFiles.cs b/DataProcessing/SEC13FFiles.cs new file mode 100644 index 0000000..fe0af1a --- /dev/null +++ b/DataProcessing/SEC13FFiles.cs @@ -0,0 +1,108 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; + +namespace QuantConnect.DataProcessing +{ + /// File helpers shared by the 13F downloader and the N-PORT crosswalk. + internal static class SEC13FFiles + { + /// + /// Streams one tab separated table out of an SEC archive, yielding the header's column index + /// map with each row. A missing table or required column throws, since either means the + /// layout moved; so does a row too short for the required columns, unless + /// asks for it to be skipped. + /// + public static IEnumerable<(Dictionary Columns, string[] Fields)> ReadTable( + ZipArchive zip, string source, string table, bool skipShortRows, params string[] required) + { + var entry = zip.Entries.FirstOrDefault(x => + string.Equals(Path.GetFileName(x.FullName), table, StringComparison.OrdinalIgnoreCase)); + if (entry == null) + { + throw new FileNotFoundException( + $"SEC13FFiles.ReadTable(): {source} carries no {table}. Entries: " + + $"{string.Join(", ", zip.Entries.Select(x => x.FullName))}"); + } + + using var reader = new StreamReader(entry.Open()); + + var header = reader.ReadLine(); + if (header == null) + { + throw new InvalidDataException($"SEC13FFiles.ReadTable(): {source} {table} is empty"); + } + + var columns = new Dictionary(StringComparer.OrdinalIgnoreCase); + var names = header.Split('\t'); + for (var i = 0; i < names.Length; i++) + { + columns[names[i].Trim()] = i; + } + + var missing = required.Where(x => !columns.ContainsKey(x)).ToList(); + if (missing.Count > 0) + { + throw new InvalidDataException( + $"SEC13FFiles.ReadTable(): {source} {table} is missing {string.Join(", ", missing)}. Header: {header}"); + } + + var maximum = required.Max(x => columns[x]); + string line; + while ((line = reader.ReadLine()) != null) + { + if (line.Length == 0) + { + continue; + } + + var fields = line.Split('\t'); + if (fields.Length <= maximum) + { + if (skipShortRows) + { + continue; + } + + throw new InvalidDataException( + $"SEC13FFiles.ReadTable(): {source} {table} row holds {fields.Length} fields, " + + $"fewer than the {maximum + 1} the required columns need: {line}"); + } + + yield return (columns, fields); + } + } + + /// + /// Writes a file through a temporary sibling that is moved into place at the end, so an + /// interrupted run never leaves a half written file for the next one to read. + /// + public static void WriteThenMove(string path, Action write) + { + var temporaryPath = path + ".tmp"; + using (var stream = File.Create(temporaryPath)) + { + write(stream); + } + + File.Move(temporaryPath, path, overwrite: true); + } + } +} diff --git a/DataProcessing/SEC13FTickerCrosswalk.cs b/DataProcessing/SEC13FTickerCrosswalk.cs new file mode 100644 index 0000000..4b0ca69 --- /dev/null +++ b/DataProcessing/SEC13FTickerCrosswalk.cs @@ -0,0 +1,297 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text.RegularExpressions; +using QuantConnect.Logging; + +namespace QuantConnect.DataProcessing +{ + /// + /// CUSIP to ticker crosswalk built from the SEC Form N-PORT data sets, used as the last step of + /// the 13F identity chain. + /// + /// A 13F reports a CUSIP and nothing else, and LEAN's security database does not carry every + /// issuer: it misses names as large as Alphabet, and for foreign issuers filing under a CINS the + /// arithmetic US ISIN is wrong by construction. N-PORT publishes, for the same securities, both + /// the CUSIP and the ticker the fund reported, so joining its tables yields a CUSIP to ticker + /// map made entirely out of filings. On the March to May 2026 13F window it covers 98.1 percent + /// of reported value, against 88.9 percent for the security database steps on their own. + /// + /// The ticker is free text written by fund administrators ("GOOGL", "GOOGL US", "goog"), so it + /// is normalised and voted on across every fund that reported the security. Each entry keeps the + /// day its data set begins, because a ticker only names a security on a date: META named a + /// Roundhill ETF from June 2021 to January 2022, before Facebook took it. + /// + /// N-PORT begins in late 2019, so a security that stopped trading before then never appears + /// here and still depends on the security database. + /// + public static class SEC13FTickerCrosswalk + { + private const string NPortUrlFormat = + "https://www.sec.gov/files/dera/data/form-n-port-data-sets/{0}q{1}_nport.zip"; + + private const string HoldingTable = "FUND_REPORTED_HOLDING.tsv"; + private const string IdentifierTable = "IDENTIFIERS.tsv"; + + /// + /// The file the built map is cached in, so the next run does not re-download gigabytes. Not + /// a .csv, so nothing that walks the security files mistakes it for one. + /// + public const string CacheFileName = "nport-crosswalk.txt"; + + /// The ticker the funds reported for a CUSIP, and the first day of the data set it came from. + public readonly record struct Entry(string Ticker, DateTime Observed); + + /// + /// Tickers arrive with a venue suffix in the Bloomberg style ("GOOGL US"), in lower case, + /// and as the literal "N/A". Strip the suffix, upper case the rest and accept only what + /// looks like a US equity ticker. + /// + private static readonly Regex VenueSuffix = new(@"\s+[A-Z]{2}$", RegexOptions.Compiled); + private static readonly Regex TickerShape = new(@"^[A-Z.\-]{1,8}$", RegexOptions.Compiled); + + /// + /// Loads the crosswalk cached in , building it from the + /// newest N-PORT data sets when the cache does not cover them and + /// writing the result to . says + /// whether a data set is published and puts one on disk, both + /// through the caller's rate limit and retries. + /// + public static Dictionary Load(string readDirectory, string writeDirectory, int quarters, + Func exists, Func download) + { + var cached = ReadCache(Path.Combine(readDirectory, CacheFileName), out var cachedQuarters); + + var missing = FindAvailableQuarters(quarters, exists) + .Where(quarter => !cachedQuarters.Contains(QuarterKey(quarter))) + .ToList(); + + if (missing.Count == 0) + { + // Written back all the same, so the published folder always carries the map the data was + // built with. A run that found it complete used to leave it out of its output, and the + // next daily run rebuilt it from 1.8 GB of N-PORT, possibly with a newer quarter. + WriteCache(Path.Combine(writeDirectory, CacheFileName), cached, cachedQuarters); + Log.Trace($"SEC13FTickerCrosswalk.Load(): cache holds {cached.Count} CUSIPs, nothing to fetch"); + return cached; + } + + var downloads = new List(); + foreach (var (year, quarter) in missing) + { + var path = download(Url(year, quarter), $"{QuarterKey((year, quarter))}_nport.zip"); + downloads.Add(path); + + Fold(cached, QuarterStart(year, quarter), BuildFromQuarter(path, year, quarter)); + cachedQuarters.Add(QuarterKey((year, quarter))); + } + + WriteCache(Path.Combine(writeDirectory, CacheFileName), cached, cachedQuarters); + + // The derived map is what is kept; each archive is several hundred megabytes. + foreach (var path in downloads) + { + File.Delete(path); + } + + Log.Trace($"SEC13FTickerCrosswalk.Load(): {cached.Count} CUSIPs after folding in " + + $"{string.Join(", ", missing.Select(QuarterKey))}"); + return cached; + } + + /// + /// Folds one data set's tickers into the map. The newest observation of a CUSIP wins whatever + /// order the data sets arrive in: a first build walks back from today, while a refresh adds + /// one newer quarter on top of the cache. + /// + internal static void Fold(Dictionary map, DateTime observed, + IEnumerable> tickers) + { + foreach (var (cusip, ticker) in tickers) + { + if (!map.TryGetValue(cusip, out var held) || held.Observed <= observed) + { + map[cusip] = new Entry(ticker, observed); + } + } + } + + /// Walks back from the current quarter until it has found the requested number of data sets. + private static List<(int Year, int Quarter)> FindAvailableQuarters(int quarters, Func exists) + { + var found = new List<(int, int)>(); + var probe = DateTime.UtcNow; + + // Twelve quarters of lookback is far more than the publication lag and stops the probe + // from walking to 2019 if the SEC ever moves the files. + for (var attempt = 0; attempt < 12 && found.Count < quarters; attempt++, probe = probe.AddMonths(-3)) + { + var year = probe.Year; + var quarter = (probe.Month - 1) / 3 + 1; + if (exists(Url(year, quarter))) + { + found.Add((year, quarter)); + } + } + + if (found.Count == 0) + { + throw new InvalidOperationException( + "SEC13FTickerCrosswalk.FindAvailableQuarters(): no N-PORT data set is published"); + } + + return found; + } + + /// + /// Builds the map for one quarter. Two passes over the archive, because the tables join on + /// HOLDING_ID and neither is sorted: the first collects the identifier rows that carry a + /// ticker, only 6.9 percent of them, and the second votes those tickers onto the CUSIP each + /// holding belongs to. + /// + private static IEnumerable> BuildFromQuarter(string path, int year, int quarter) + { + using var archive = ZipFile.OpenRead(path); + + var tickersByHolding = new Dictionary(StringComparer.Ordinal); + foreach (var fields in ReadTable(archive, IdentifierTable, "HOLDING_ID", "IDENTIFIER_TICKER")) + { + var ticker = Normalize(fields[1]); + if (ticker != null) + { + tickersByHolding[fields[0]] = ticker; + } + } + + var votes = new Dictionary>(StringComparer.Ordinal); + foreach (var fields in ReadTable(archive, HoldingTable, "HOLDING_ID", "ISSUER_CUSIP")) + { + if (!tickersByHolding.TryGetValue(fields[0], out var ticker)) + { + continue; + } + + var cusip = fields[1].Trim().ToUpperInvariant(); + + // Foreign issuers without a CUSIP are masked as all zeros in this data set. + if (cusip.Length != 9 || cusip == "000000000") + { + continue; + } + + if (!votes.TryGetValue(cusip, out var tally)) + { + votes[cusip] = tally = new Dictionary(StringComparer.Ordinal); + } + + tally.TryGetValue(ticker, out var count); + tally[ticker] = count + 1; + } + + Log.Trace($"SEC13FTickerCrosswalk.BuildFromQuarter(): {year}Q{quarter}: " + + $"{tickersByHolding.Count} holdings carried a ticker, {votes.Count} CUSIPs resolved"); + + return votes.Select(pair => new KeyValuePair( + pair.Key, + pair.Value.OrderByDescending(vote => vote.Value).ThenBy(vote => vote.Key, StringComparer.Ordinal).First().Key)) + .ToList(); + } + + /// + /// The requested columns of one N-PORT table. Short rows are skipped rather than thrown on, + /// so one malformed holding cannot stop the whole build. + /// + private static IEnumerable ReadTable(ZipArchive archive, string table, params string[] wanted) + { + return SEC13FFiles.ReadTable(archive, "N-PORT", table, skipShortRows: true, wanted) + .Select(row => wanted.Select(column => row.Fields[row.Columns[column]]).ToArray()); + } + + /// Cleans one raw ticker, returning null when it is not usable. + internal static string Normalize(string raw) + { + var ticker = VenueSuffix.Replace(raw.Trim().ToUpperInvariant(), string.Empty); + return ticker.Length > 0 && ticker != "N/A" && TickerShape.IsMatch(ticker) ? ticker : null; + } + + private static string Url(int year, int quarter) + => string.Format(CultureInfo.InvariantCulture, NPortUrlFormat, year, quarter); + + private static string QuarterKey((int Year, int Quarter) quarter) + => $"{quarter.Year}q{quarter.Quarter}"; + + /// + /// The day a data set's tickers are taken to be observed on: its quarter's first day. A ticker + /// renamed within that quarter fails the traded-under check in the downloader, so its CUSIP + /// stays unresolved until a newer quarter is folded in. Rare, and it never resolves wrongly. + /// + private static DateTime QuarterStart(int year, int quarter) + => new(year, quarter * 3 - 2, 1); + + private static Dictionary ReadCache(string path, out HashSet quarters) + { + var map = new Dictionary(StringComparer.Ordinal); + quarters = new HashSet(StringComparer.OrdinalIgnoreCase); + + if (!File.Exists(path)) + { + return map; + } + + foreach (var line in File.ReadLines(path)) + { + if (line.StartsWith("#", StringComparison.Ordinal)) + { + quarters.UnionWith(line.TrimStart('#').Split(',', StringSplitOptions.RemoveEmptyEntries) + .Select(value => value.Trim())); + continue; + } + + var fields = line.Split(','); + if (fields.Length == 3 && fields[0].Length == 9) + { + map[fields[0]] = new Entry(fields[1], + DateTime.ParseExact(fields[2], DateFormat.EightCharacter, CultureInfo.InvariantCulture)); + } + } + + return map; + } + + private static void WriteCache(string path, Dictionary map, HashSet quarters) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)); + + var lines = new List { "#" + string.Join(",", quarters.OrderBy(value => value, StringComparer.Ordinal)) }; + lines.AddRange(map.OrderBy(pair => pair.Key, StringComparer.Ordinal) + .Select(pair => $"{pair.Key},{pair.Value.Ticker},{pair.Value.Observed.ToString(DateFormat.EightCharacter, CultureInfo.InvariantCulture)}")); + + SEC13FFiles.WriteThenMove(path, stream => + { + using var writer = new StreamWriter(stream, leaveOpen: true); + foreach (var line in lines) + { + writer.WriteLine(line); + } + }); + } + } +} diff --git a/QuantConnect.DataSource.csproj b/QuantConnect.DataSource.csproj index 70a709c..e5bbc05 100644 --- a/QuantConnect.DataSource.csproj +++ b/QuantConnect.DataSource.csproj @@ -1,4 +1,4 @@ - + net10.0 @@ -22,12 +22,10 @@ - - - - - - + + + diff --git a/SEC13F.cs b/SEC13F.cs new file mode 100644 index 0000000..591e979 --- /dev/null +++ b/SEC13F.cs @@ -0,0 +1,234 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using NodaTime; +using ProtoBuf; +using QuantConnect.Data; +using QuantConnect.Python; +using QuantConnect.Util; + +namespace QuantConnect.DataSource +{ + /// + /// SEC Form 13F institutional holdings for one security, as they stood on the day the point was + /// published. Institutional investment managers exercising discretion over at least 100 million + /// dollars must file a Form 13F within 45 days of quarter end, listing the covered securities + /// they hold. This data point answers how much of a name institutions hold and how many of them + /// hold it. + /// + /// One release can restate SEVERAL quarters of a security at once, because late filings and + /// amendments keep arriving, so a point carries them all in , one + /// per reported quarter, oldest first. Seven percent of the points + /// in the history carry more than one, up to 75. LEAN hands an algorithm a single data point + /// per security per timestamp, so a line per quarter would have reached the algorithm one at a + /// time and the rest would have been dropped. + /// + /// Every value is CUMULATIVE for its quarter: it counts every filing for that quarter that was + /// public by the point timestamp, not only the filings made that day. The managers of a single + /// quarter file over roughly fifty different days, so a per-day figure would answer a question + /// nobody asks: on the busiest day of the March 2026 quarter 602 managers reported Apple for + /// the first time, while 5,920 had reported it by then. + /// + /// Raw sums only. Quarter-over-quarter change, percentage of shares outstanding and + /// concentration ratios are all derivable and are left to the algorithm. + /// + [ProtoContract(SkipConstructor = true, IgnoreListHandling = true)] + public class SEC13F : BaseData, IEnumerable + { + // The frozen layout of a line: the release stamp, how many quarters it carries, and then + // that many groups of columns. The group width is read from the line rather than assumed, + // so a measure appended in a later revision leaves every existing file readable. + private const int HeaderColumns = 2; + internal const int GroupColumns = 10; + internal const string ReleaseFormat = "yyyyMMdd HH:mm"; + + private DateTime? _endTime; + + /// + /// When the release became available. The per security files carry the release stamp in + /// their first column, so it equals Time there; the universe files are named by their date + /// and set it to the release time of that date. + /// + [ProtoMember(11)] + public override DateTime EndTime + { + get => _endTime ?? Time; + set => _endTime = value; + } + + /// + /// The quarters this release restated, oldest first. Never empty: a security with nothing + /// to report on a day has no point at all that day. + /// + [ProtoMember(10)] + public List Holdings { get; set; } = new(); + + /// + /// The quarter the most managers have reported, which is the most complete reading the + /// point carries. While a new quarter fills in, over roughly six weeks, the quarter before + /// it still holds more managers and is the one that answers how widely a name is held. + /// This is the reading whose fields the pandas DataFrame carries as columns. + /// + public SEC13FHolding MostReported => Holdings + .OrderByDescending(holding => holding.Holders ?? 0m) + .ThenByDescending(holding => holding.PeriodEnd) + .FirstOrDefault(); + + /// + /// The most recent quarter this release restated, which during a quarter handover is the + /// one still filling in. Kept out of the DataFrame: a second expanded reading would prefix + /// every column with the name of the property it came from. + /// + [PandasIgnore] + public SEC13FHolding Latest => Holdings.Count == 0 ? null : Holdings[Holdings.Count - 1]; + + /// Name of the dataset folder under alternative/sec/, which is where its files live. + public static string ReportFolder => "13f"; + + /// + /// Time of day a release date points become available. EDGAR lists a day of filings at + /// about 22:05 ET, the daily job reads them at 01:00 ET the next day and publishes by + /// 03:00, so a filing is released the day after its filing date at this time, before the + /// market opens. + /// + public static TimeSpan ReleaseTimeOfDay => TimeSpan.FromHours(3); + + /// Location of the source file: alternative/sec/13f/{ticker}.csv + public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode) + { + return new SubscriptionDataSource( + Path.Combine( + Globals.DataFolder, + "alternative", + "sec", + ReportFolder, + $"{config.Symbol.Value.ToLowerInvariant()}.csv" + ), + SubscriptionTransportMedium.LocalFile, + FileFormat.Csv + ); + } + + /// Parses one line, which carries every quarter the release restated. + public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) + { + var point = Parse(line); + if (point == null) + { + return null; + } + + point.Symbol = config.Symbol; + foreach (var holding in point.Holdings) + { + holding.Symbol = config.Symbol; + } + + return point; + } + + /// + /// Parses a line into a point that has no symbol yet, which is what the universe file needs + /// once it has read the identifier of its own line. A malformed line gives null rather than + /// throwing: an exception out of Reader ends the algorithm. + /// + internal static SEC13F Parse(string line) + { + var csv = line.Split(','); + if (csv.Length < HeaderColumns + GroupColumns || + !int.TryParse(csv[1], NumberStyles.Integer, CultureInfo.InvariantCulture, out var count) || + count <= 0) + { + return null; + } + + var width = (csv.Length - HeaderColumns) / count; + if (width < GroupColumns) + { + return null; + } + + var point = new SEC13F + { + Time = DateTime.ParseExact(csv[0], ReleaseFormat, CultureInfo.InvariantCulture) + }; + + for (var i = 0; i < count; i++) + { + point.Holdings.Add(SEC13FHolding.Parse(csv, HeaderColumns + i * width)); + } + + point.Value = point.MostReported?.HoldingValue ?? 0m; + return point; + } + + /// Data time zone (Eastern, the SEC filing time zone). + public override DateTimeZone DataTimeZone() => TimeZones.NewYork; + + /// Supported resolutions (Daily only, the quarterly cadence is modeled as Daily). + public override List SupportedResolutions() => DailyResolution; + + /// Default resolution. + public override Resolution DefaultResolution() => Resolution.Daily; + + /// Sparse data: one file per security, so missing-file logs are suppressed. + public override bool IsSparseData() => true; + + /// Linked to Equities, so renames and delistings are applied via map files. + public override bool RequiresMapping() => true; + + /// + /// Iterates the quarters, which is what lets pandas flatten a point into rows. A point that + /// arrived live was deserialized from protobuf, which carries the symbol of the point out of + /// band and leaves its quarters without one, so it is filled in here rather than left null + /// for the flattening to read. + /// + public IEnumerator GetEnumerator() + { + foreach (var holding in Holdings) + { + holding.Symbol ??= Symbol; + yield return holding; + } + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + /// Creates a copy of the instance. + public override BaseData Clone() + { + return new SEC13F + { + Symbol = Symbol, + Time = Time, + EndTime = EndTime, + Value = Value, + Holdings = Holdings.Select(holding => holding.Clone()).ToList() + }; + } + + /// String representation for debugging. + public override string ToString() + { + return $"{Symbol}({Time:yyyyMMdd}) :: [{string.Join(", ", Holdings)}]"; + } + } +} diff --git a/SEC13FAlgorithm.cs b/SEC13FAlgorithm.cs new file mode 100644 index 0000000..5ad72ba --- /dev/null +++ b/SEC13FAlgorithm.cs @@ -0,0 +1,145 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Linq; +using System.Collections.Generic; +using QuantConnect.Data; +using QuantConnect.Orders; +using QuantConnect.Algorithm; +using QuantConnect.DataSource; + +namespace QuantConnect.DataLibrary.Tests +{ + /// + /// Example algorithm using the SEC Form 13F institutional holdings dataset as a source of alpha. + /// It subscribes the dataset on three large, long-listed US equities, reads the aggregated + /// institutional position on each (number of reporting managers, shares held and their market + /// value), and holds the name the most institutions report. + /// + /// The 13F symbols returned by AddData are signals, not tradeable securities, so every name is + /// added twice: once as the tradeable equity and once as the custom data subscribed on it. + /// + public class SEC13FAlgorithm : QCAlgorithm + { + /// Tradeable equity symbol, keyed by the 13F data symbol subscribed on it. + private readonly Dictionary _equityByDataSymbol = new Dictionary(); + + /// Latest institutional breadth per equity, read off the most complete quarter. + private readonly Dictionary _holdersByEquity = new Dictionary(); + + private Symbol _invested; + + /// + /// Initialise the data and resolution required, as well as the cash and start-end dates. + /// + public override void Initialize() + { + // The dataset runs 2013-05-20 to 2026-05-29, which is where the SEC's structured 13F + // tables begin. This window sits inside it and holds one whole quarterly cycle: the + // September 2020 quarter fills in from a handful of managers in early October to + // thousands by the middle of November, while the June quarter it replaces finishes. + // That handover is the behaviour the cumulative shape exists for. + SetStartDate(2020, 10, 1); + SetEndDate(2020, 12, 31); + SetCash(100000); + + // AAPL is one of the names whose aggregation was validated against published + // institutional ownership. GOOGL is here on purpose too: its CUSIP is not in the + // security database, so it is reachable only through the N-PORT ticker crosswalk, and + // seeing it arrive proves the whole identity chain rather than just its first step. + foreach (var ticker in new[] { "AAPL", "GOOGL", "SPY" }) + { + var equity = AddEquity(ticker, Resolution.Daily).Symbol; + var dataSymbol = AddData(equity).Symbol; + _equityByDataSymbol[dataSymbol] = equity; + } + } + + /// + /// OnData event is the primary entry point for your algorithm. Each new data point is here. + /// + /// Slice object keyed by symbol containing the data + public override void OnData(Slice slice) + { + foreach (var kvp in _equityByDataSymbol) + { + if (!slice.ContainsKey(kvp.Key)) + { + continue; + } + + // A release can restate several quarters of the same security at once and the point + // carries every one of them, so indexing the Slice by symbol loses nothing. + SEC13F point = slice[kvp.Key]; + var equity = kvp.Value; + + // Time is when the point was published, 03:00 ET the day after the filing date, + // while PeriodEnd is the quarter a reading describes, typically 45 to 135 days + // earlier. Every value is cumulative for its own PeriodEnd: it counts every manager + // that had reported the security for that quarter by the publication time. + foreach (var holding in point.Holdings) + { + Log($"{Time:yyyy-MM-dd} {equity.Value} - Period: {holding.Quarter} ({holding.PeriodEnd:yyyy-MM-dd}), Holders: {holding.Holders}, Shares: {holding.Shares}, HoldingValue: {holding.HoldingValue}"); + } + + // MostReported is the quarter the most managers have reported, so a new quarter that + // is still filling in cannot hide the finished one it replaces: the same rule the + // universe file applies. An algorithm chasing the freshest read would take Latest. + var mostReported = point.MostReported; + if (mostReported.Holders.HasValue) + { + _holdersByEquity[equity] = mostReported.Holders.Value; + } + } + + if (_holdersByEquity.Count == 0) + { + return; + } + + // Hold the name the most institutions report. Rebalancing only when the leader changes + // keeps the demo down to a handful of orders. + var leader = _holdersByEquity + .OrderByDescending(kvp => kvp.Value) + .ThenBy(kvp => kvp.Key.Value) + .First() + .Key; + + // A 13F point lands before the open on the day after its filing, which is not necessarily + // a day the equity prints a bar, so the order waits for a price rather than firing + // against a stale one. + if (leader == _invested || !slice.Bars.ContainsKey(leader)) + { + return; + } + + _invested = leader; + Log($"{Time:yyyy-MM-dd} most widely held name is now {leader.Value}, rotating into it"); + SetHoldings(leader, 1, liquidateExistingHoldings: true); + } + + /// + /// Order fill event handler. + /// + /// Order event details + public override void OnOrderEvent(OrderEvent orderEvent) + { + if (orderEvent.Status == OrderStatus.Filled) + { + Debug($"{Time} - Filled: {orderEvent.Symbol} {orderEvent.FillQuantity}"); + } + } + } +} diff --git a/SEC13FAlgorithm.py b/SEC13FAlgorithm.py new file mode 100644 index 0000000..5bdcf2f --- /dev/null +++ b/SEC13FAlgorithm.py @@ -0,0 +1,97 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from AlgorithmImports import * +from QuantConnect.DataSource import * + + +class SEC13FAlgorithm(QCAlgorithm): + '''Example algorithm using the SEC Form 13F institutional holdings dataset as a source of alpha. + It subscribes the dataset on three large, long-listed US equities, reads the aggregated + institutional position on each (number of reporting managers, shares held and their market + value), and holds the name the most institutions report. + + The 13F symbols returned by add_data are signals, not tradeable securities, so every name is + added twice: once as the tradeable equity and once as the custom data subscribed on it.''' + + def initialize(self) -> None: + # The dataset runs 2013-05-20 to 2026-05-29, which is where the SEC's structured 13F + # tables begin. This window sits inside it and holds one whole quarterly cycle: the + # September 2020 quarter fills in from a handful of managers in early October to + # thousands by the middle of November, while the June quarter it replaces finishes. + # That handover is the behaviour the cumulative shape exists for. + self.set_start_date(2020, 10, 1) + self.set_end_date(2020, 12, 31) + self.set_cash(100000) + + # Tradeable equity symbol, keyed by the 13F data symbol subscribed on it. + self._equity_by_data_symbol = {} + + # Latest institutional breadth per equity, read off the most complete quarter. + self._holders_by_equity = {} + + self._invested = None + + # AAPL is one of the names whose aggregation was validated against published institutional + # ownership. GOOGL is here on purpose too: its CUSIP is not in the security database, so it + # is reachable only through the N-PORT ticker crosswalk, and seeing it arrive proves the + # whole identity chain rather than just its first step. + for ticker in ["AAPL", "GOOGL", "SPY"]: + equity = self.add_equity(ticker, Resolution.DAILY).symbol + data_symbol = self.add_data(SEC13F, equity).symbol + self._equity_by_data_symbol[data_symbol] = equity + + def on_data(self, slice: Slice) -> None: + for data_symbol, equity in self._equity_by_data_symbol.items(): + if data_symbol not in slice: + continue + + # A release can restate several quarters of the same security at once and the point + # carries every one of them, so indexing the Slice by symbol loses nothing. + point = slice[data_symbol] + + # time is when the point was published, 03:00 ET the day after the filing date, while + # period_end is the quarter a reading describes, typically 45 to 135 days earlier. + # Every value is cumulative for its own period_end: it counts every manager that had + # reported the security for that quarter by the publication time. + for holding in point.holdings: + self.log(f"{self.time:%Y-%m-%d} {equity.value} - Period: {holding.quarter} ({holding.period_end:%Y-%m-%d}), Holders: {holding.holders}, Shares: {holding.shares}, HoldingValue: {holding.holding_value}") + + # most_reported is the quarter the most managers have reported, so a new quarter that is + # still filling in cannot hide the finished one it replaces: the same rule the universe + # file applies. An algorithm chasing the freshest read would take latest. + most_reported = point.most_reported + if most_reported.holders is not None: + self._holders_by_equity[equity] = most_reported.holders + + if not self._holders_by_equity: + return + + # Hold the name the most institutions report. Rebalancing only when the leader changes keeps + # the demo down to a handful of orders. + leader = sorted(self._holders_by_equity.items(), + key=lambda kvp: (-kvp[1], kvp[0].value))[0][0] + + # A 13F point lands before the open on the day after its filing, which is not necessarily a + # day the equity prints a bar, so the order waits for a price rather than firing against a + # stale one. + if leader == self._invested or leader not in slice.bars: + return + + self._invested = leader + self.log(f"{self.time:%Y-%m-%d} most widely held name is now {leader.value}, rotating into it") + self.set_holdings(leader, 1, liquidate_existing_holdings=True) + + def on_order_event(self, order_event: OrderEvent) -> None: + if order_event.status == OrderStatus.FILLED: + self.debug(f"{self.time} - Filled: {order_event.symbol} {order_event.fill_quantity}") diff --git a/SEC13FHolding.cs b/SEC13FHolding.cs new file mode 100644 index 0000000..5ccb65d --- /dev/null +++ b/SEC13FHolding.cs @@ -0,0 +1,139 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Globalization; +using ProtoBuf; +using QuantConnect.Data; +using QuantConnect.Python; +using QuantConnect.Util; + +namespace QuantConnect.DataSource +{ + /// + /// One reported quarter of a point: every 13F filing for that quarter that + /// was public by the point's timestamp, summed. Deliberately not a , + /// so that it can ride inside the point: the data distributor registers every BaseData type + /// exposing a DataSourceId as a protobuf sub type, and a nested sub type cannot be serialized. + /// + [ProtoContract(SkipConstructor = true)] + public class SEC13FHolding : ISymbolProvider + { + /// + /// The security the quarter is reported for, which is the point's own symbol. Carried so + /// that pandas can flatten a point into one row per quarter, and kept out of the frame + /// itself, where it would collide with the index. + /// + [PandasIgnore] + public Symbol Symbol { get; set; } + + /// + /// The quarter the positions are reported for, which is the SEC PERIODOFREPORT. It is + /// carried rather than derived from the point's timestamp because the two are unrelated: + /// late filings and amendments mean one publication day carries several different reported + /// quarters, and the gap between them runs from zero to years. + /// + [ProtoMember(1)] + public DateTime PeriodEnd { get; set; } + + /// + /// The reported quarter as a label, such as 2020Q2. Derived from and + /// never stored, so it cannot disagree with it. The year comes first so that sorting the + /// label sorts the quarters. + /// + public string Quarter => $"{PeriodEnd.Year}Q{(PeriodEnd.Month - 1) / 3 + 1}"; + + /// + /// Number of distinct managers, counted by filer CIK, that have reported the security for + /// by the point's timestamp. A manager counts once however many + /// filings it makes, however many of the issuer's CUSIPs it names, and whether the filing + /// that first names the security is an original or an amendment. + /// + [ProtoMember(2)] + public decimal? Holders { get; set; } + + /// + /// Shares held, summed over every reported line with share-type SH and no option flag. + /// Principal amounts of debt (PRN) and option lines are deliberately excluded. + /// + [ProtoMember(3)] + public decimal? Shares { get; set; } + + /// + /// Market value of the shares above as reported by the managers, in whole dollars. The + /// most reported quarter's value is also the point's Value. + /// + [ProtoMember(4)] + public decimal? HoldingValue { get; set; } + + /// Shares underlying reported call positions, kept separate from Shares. + [ProtoMember(5)] + public decimal? CallShares { get; set; } + + /// Shares underlying reported put positions, kept separate from Shares. + [ProtoMember(6)] + public decimal? PutShares { get; set; } + + /// Principal amount of debt instruments reported for the security (share-type PRN). + [ProtoMember(7)] + public decimal? PrincipalValue { get; set; } + + /// Shares over which the reporting managers hold sole voting authority. + [ProtoMember(8)] + public decimal? VotingSole { get; set; } + + /// Shares over which the reporting managers share voting authority. + [ProtoMember(9)] + public decimal? VotingShared { get; set; } + + /// + /// True when at least one contributing filing withheld positions under confidential + /// treatment. Such a filing is incomplete by design and the withheld positions surface in + /// a later publication, so the flag is carried rather than silently ignored. + /// + [ProtoMember(10)] + public bool ConfidentialOmitted { get; set; } + + /// Reads one group of columns, starting at the reported quarter. + internal static SEC13FHolding Parse(string[] csv, int offset) + { + return new SEC13FHolding + { + PeriodEnd = DateTime.ParseExact(csv[offset], "yyyyMMdd", CultureInfo.InvariantCulture), + Holders = ParseMeasure(csv[offset + 1]), + Shares = ParseMeasure(csv[offset + 2]), + HoldingValue = ParseMeasure(csv[offset + 3]), + CallShares = ParseMeasure(csv[offset + 4]), + PutShares = ParseMeasure(csv[offset + 5]), + PrincipalValue = ParseMeasure(csv[offset + 6]), + VotingSole = ParseMeasure(csv[offset + 7]), + VotingShared = ParseMeasure(csv[offset + 8]), + ConfidentialOmitted = csv[offset + 9] == "1" + }; + } + + /// Parses one measure column, where an empty field is an absent reading. + internal static decimal? ParseMeasure(string value) + { + return value.IfNotNullOrEmpty(s => decimal.Parse(s, NumberStyles.Any, CultureInfo.InvariantCulture)); + } + + /// Creates a copy of the instance. + public SEC13FHolding Clone() => (SEC13FHolding)MemberwiseClone(); + + /// String representation for debugging. + public override string ToString() => $"{Quarter}: {Holders} holders, {Shares} shares"; + } +} diff --git a/SEC13FUniverse.cs b/SEC13FUniverse.cs new file mode 100644 index 0000000..c071c93 --- /dev/null +++ b/SEC13FUniverse.cs @@ -0,0 +1,120 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using NodaTime; +using QuantConnect.Data; +using QuantConnect.Data.UniverseSelection; +using QuantConnect.Util; + +namespace QuantConnect.DataSource +{ + /// + /// Universe selection data for the SEC Form 13F institutional holdings dataset: one + /// per security per release date, so the whole cross-section can be ranked + /// or filtered by institutional ownership without subscribing security by security. + /// + /// The file is keyed by release date, not by reported quarter: a single release date carries + /// filings for several quarters, because late filers and amendments keep arriving. Each + /// security's quarters travel together in its , so the selection + /// function sees one record per security rather than one per quarter. + /// + /// Each business day carries every security's live quarters forward, not only the ones filed + /// that day. A quarter stops being carried once the next quarter's 45 day filing deadline has + /// passed, and a security leaves after its delisting date, so a name nobody reports any more + /// drops out rather than sitting in every file with its last quarter. + /// + public class SEC13FUniverse : BaseDataCollection + { + // The line: the identifier, the ticker, and then a SEC13F point in its own layout. + private const int IdentifierColumns = 2; + + /// Location of the universe file: alternative/sec/13f/universe/{yyyyMMdd}.csv + public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode) + { + return new SubscriptionDataSource( + System.IO.Path.Combine( + Globals.DataFolder, + "alternative", + "sec", + SEC13F.ReportFolder, + "universe", + $"{date.ToStringInvariant(DateFormat.EightCharacter)}.csv" + ), + SubscriptionTransportMedium.LocalFile, + FileFormat.FoldingCollection + ); + } + + /// Parses one universe line into the security's point for the release date. + public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) + { + var csv = line.Split(','); + if (csv.Length < IdentifierColumns) + { + return null; + } + + // The release stamp is the file date rather than a column, so the point is read with a + // stamp of its own put in front of the rest of the line. The file day stays the point + // Time and the release time becomes its EndTime, which is when an algorithm can act on + // it: the engine walks a universe file by the day it is named after. + var stamp = date.ToStringInvariant(SEC13F.ReleaseFormat); + var point = SEC13F.Parse($"{stamp},{string.Join(",", csv.Skip(IdentifierColumns))}"); + if (point == null) + { + return null; + } + + point.EndTime = date + SEC13F.ReleaseTimeOfDay; + + point.Symbol = new Symbol(SecurityIdentifier.Parse(csv[0]), csv[1]); + foreach (var holding in point.Holdings) + { + holding.Symbol = point.Symbol; + } + + return point; + } + + /// Sparse: universe files exist only on release dates. + public override bool IsSparseData() => true; + + /// Default resolution. + public override Resolution DefaultResolution() => Resolution.Daily; + + /// Supported resolutions (Daily only). + public override List SupportedResolutions() => DailyResolution; + + /// Data time zone (Eastern, the SEC filing time zone). + public override DateTimeZone DataTimeZone() => TimeZones.NewYork; + + /// Creates a copy of the instance. + public override BaseData Clone() + { + return new SEC13FUniverse + { + Symbol = Symbol, + Time = Time, + EndTime = EndTime, + Underlying = Underlying, + FilteredContracts = FilteredContracts, + Data = Data?.Select(point => point.Clone()).ToList() + }; + } + } +} diff --git a/SEC13FUniverseSelectionAlgorithm.cs b/SEC13FUniverseSelectionAlgorithm.cs new file mode 100644 index 0000000..c268563 --- /dev/null +++ b/SEC13FUniverseSelectionAlgorithm.cs @@ -0,0 +1,139 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Linq; +using System.Collections.Generic; +using QuantConnect.Data; +using QuantConnect.Algorithm; +using QuantConnect.DataSource; +using QuantConnect.Data.UniverseSelection; +using QuantConnect.Algorithm.Framework.Portfolio; + +namespace QuantConnect.DataLibrary.Tests +{ + /// + /// Example algorithm demonstrating universe selection over the SEC Form 13F institutional + /// holdings dataset: on every release date it ranks the cross-section by the number of + /// institutional managers reporting each security and holds an equal-weight basket of the most + /// widely held names. + /// + /// The universe file is keyed by the release date rather than by the reported quarter, so a + /// single selection call mixes filings covering several different quarters. Each security + /// arrives as one record carrying its live quarters together, and PeriodEnd says which quarter + /// a reading describes: it typically sits 45 to 135 days behind the day the record arrives, + /// with late amendments arriving years later. That lag is the product, not a defect. + /// + public class SEC13FUniverseSelectionAlgorithm : QCAlgorithm + { + /// Number of names held at a time. + private const int BasketSize = 5; + + private readonly List _selected = new List(); + private bool _rebalance; + + /// + /// Initialise the data and resolution required, as well as the cash and start-end dates. + /// + public override void Initialize() + { + // Data ADDED via universe selection is added with Daily resolution. + UniverseSettings.Resolution = Resolution.Daily; + + // The universe runs 2013-05-20 to 2026-05-29, forward filled across business days. One + // quarter is enough here: it covers a whole handover, which is where the interesting + // behaviour is, and it is the same window the per-security demo reads. + SetStartDate(2020, 10, 1); + SetEndDate(2020, 12, 31); + SetCash(100000); + + AddUniverse(data => + { + // Holders counts distinct filer CIKs, so it ranks names by how broadly institutions + // own them rather than by how much money one large manager put in. + // + // Every security arrives once, with the quarter that has finished filling in and + // the one still filling travelling together in Holdings. MostReported is the one + // more managers have reported, which is the more complete picture and what a + // breadth ranking wants. An algorithm chasing the freshest read would take Latest. + var selected = (from SEC13F datum in data + let quarter = datum.MostReported + where quarter.Holders.HasValue + orderby quarter.Holders.Value descending, datum.Symbol.Value + select datum) + .Take(BasketSize) + .ToList(); + + foreach (var datum in selected) + { + var quarter = datum.MostReported; + Log($"{datum.Symbol.Value} - Period: {quarter.Quarter} ({quarter.PeriodEnd:yyyy-MM-dd}), Holders: {quarter.Holders}, HoldingValue: {quarter.HoldingValue}"); + } + + return selected.Select(datum => datum.Symbol); + }); + } + + /// + /// Event fired each time that we add/remove securities from the data feed. + /// + /// Security additions/removals for this time step + public override void OnSecuritiesChanged(SecurityChanges changes) + { + Log(changes.ToString()); + + foreach (var security in changes.RemovedSecurities) + { + _selected.Remove(security.Symbol); + } + + foreach (var security in changes.AddedSecurities) + { + _selected.Add(security.Symbol); + } + + _rebalance = true; + } + + /// + /// OnData event is the primary entry point for your algorithm. Each new data point is here. + /// + /// Slice object keyed by symbol containing the data + public override void OnData(Slice slice) + { + if (!_rebalance || _selected.Count == 0) + { + return; + } + + // A release date is not necessarily a day every name in the basket printed a bar, so + // the rebalance takes the ones that did rather than waiting for a day when all of them + // do. Blocking on the whole basket sounds safer and is not: one name that never prints, + // for a holiday or a halt or simply because it is missing from the local data, stops + // the algorithm from ever trading at all. + var priced = _selected.Where(symbol => slice.Bars.ContainsKey(symbol)).ToList(); + if (priced.Count == 0) + { + return; + } + + _rebalance = false; + + var weight = 1m / priced.Count; + SetHoldings( + priced.Select(symbol => new PortfolioTarget(symbol, weight)).ToList(), + liquidateExistingHoldings: true); + } + } +} diff --git a/SEC13FUniverseSelectionAlgorithm.py b/SEC13FUniverseSelectionAlgorithm.py new file mode 100644 index 0000000..d8a0aee --- /dev/null +++ b/SEC13FUniverseSelectionAlgorithm.py @@ -0,0 +1,96 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from AlgorithmImports import * +from QuantConnect.DataSource import * + + +class SEC13FUniverseSelectionAlgorithm(QCAlgorithm): + '''Example algorithm demonstrating universe selection over the SEC Form 13F institutional + holdings dataset: on every release date it ranks the cross-section by the number of + institutional managers reporting each security and holds an equal-weight basket of the most + widely held names. + + The universe file is keyed by the release date rather than by the reported quarter, so a single + selection call mixes filings covering several different quarters. Each security arrives as one + record carrying its live quarters together, and period_end says which quarter a reading + describes: it typically sits 45 to 135 days behind the day the record arrives, with late + amendments arriving years later. That lag is the product, not a defect.''' + + # Number of names held at a time. + BASKET_SIZE = 5 + + def initialize(self) -> None: + # Data ADDED via universe selection is added with Daily resolution. + self.universe_settings.resolution = Resolution.DAILY + + # The universe runs 2013-05-20 to 2026-05-29, forward filled across business days. One + # quarter is enough here: it covers a whole handover, which is where the interesting + # behaviour is, and it is the same window the per-security demo reads. + self.set_start_date(2020, 10, 1) + self.set_end_date(2020, 12, 31) + self.set_cash(100000) + + self._selected = [] + self._rebalance = False + + self.add_universe(SEC13FUniverse, self._select) + + def _select(self, data: List[SEC13F]) -> List[Symbol]: + # holders counts distinct filer CIKs, so it ranks names by how broadly institutions own + # them rather than by how much money one large manager put in. + # Every security arrives once, with the quarter that has finished filling in and the one + # still filling travelling together in holdings. most_reported is the one more managers + # have reported, which is the more complete picture and what a breadth ranking wants. An + # algorithm chasing the freshest read would take latest. + ranked = [datum for datum in data if datum.most_reported.holders is not None] + selected = sorted(ranked, + key=lambda datum: (-datum.most_reported.holders, + datum.symbol.value))[:self.BASKET_SIZE] + + for datum in selected: + quarter = datum.most_reported + self.log(f"{datum.symbol.value} - Period: {quarter.quarter} ({quarter.period_end:%Y-%m-%d}), Holders: {quarter.holders}, HoldingValue: {quarter.holding_value}") + + return [datum.symbol for datum in selected] + + def on_securities_changed(self, changes: SecurityChanges) -> None: + self.log(str(changes)) + + for security in changes.removed_securities: + if security.symbol in self._selected: + self._selected.remove(security.symbol) + + for security in changes.added_securities: + self._selected.append(security.symbol) + + self._rebalance = True + + def on_data(self, slice: Slice) -> None: + if not self._rebalance or not self._selected: + return + + # A release date is not necessarily a day every name in the basket printed a bar, so the + # rebalance takes the ones that did rather than waiting for a day when all of them do. + # Blocking on the whole basket sounds safer and is not: one name that never prints, for a + # holiday or a halt or simply because it is missing from the local data, stops the + # algorithm from ever trading at all. + priced = [symbol for symbol in self._selected if symbol in slice.bars] + if not priced: + return + + self._rebalance = False + + weight = 1.0 / len(priced) + self.set_holdings([PortfolioTarget(symbol, weight) for symbol in priced], + liquidate_existing_holdings=True) diff --git a/listing-about-13f.md b/listing-about-13f.md new file mode 100644 index 0000000..fd5b444 --- /dev/null +++ b/listing-about-13f.md @@ -0,0 +1,226 @@ +## Introduction + +The SEC Form 13F Institutional Holdings dataset by the U.S. Securities and Exchange Commission +tracks how much of each US Equity institutional managers report holding, from the second quarter +of 2013 to the present. Every institutional investment manager exercising discretion over at least +100 million dollars must file a Form 13F within 45 days of the end of a calendar quarter, listing +the covered securities it holds. This dataset sums those filings per security, so one data point +answers how many managers hold a name and how much of it they hold. Data arrives the morning after +each filing becomes public, which for a quarterly filing regime means a burst of activity around each +deadline and a long tail of late filers and amendments. + +The reporting lag is the product, not an inconvenience to be hidden. A position is typically 45 to +135 days old by the time it reaches the public record, with late amendments arriving years later, +so a data point is stamped with the moment it was published and carries every quarter that release +restated, each with its own `PeriodEnd`. Measured across 11,761 filings in one window, the lag from +the reported quarter end to the filing date runs minimum 0 days, p10 16, median 42, p90 48, +maximum 6,596, with 10.4 percent of filings arriving later than the 45 day deadline. +Delivering a position on the quarter end it +describes would inject every day of that gap as look-ahead, so LEAN delivers it when the dataset +could first publish it: EDGAR lists a day's filings at about 22:05 ET, the daily job reads them after +midnight, and every row is stamped at 03:00 ET the day after its filing date, before the market +opens. History and live carry the same stamp, so a backtest never reads a filing the live job did +not have yet. + +Every value is cumulative for its `PeriodEnd`. The managers of a single quarter file across roughly +fifty different days, so a data point counts every filing for that quarter that was public by its +timestamp rather than only the ones made that day. Reading Apple on the busiest day of the March +2026 quarter gives 5,920 reporting managers and 9,097,811,804 shares, not the 602 managers who +reported it for the first time that day. + +One release can restate several quarters of the same security at once, because late filings and +amendments keep arriving, so a data point carries them all in `Holdings`, one reading per reported +quarter, oldest first. Seven percent of the points in the history carry more than one, up to 75. +Shipping a point per quarter instead would have lost all but one of them, since LEAN delivers a +single data point per security per timestamp. `MostReported` picks the quarter the most managers +have reported, which during a quarter handover is the finished quarter rather than the thin new one, +and `Latest` picks the newest. + +## About the Provider + +The [U.S. Securities and Exchange Commission](https://www.sec.gov) is the federal agency that +regulates the US securities markets. Form 13F is filed through EDGAR, the SEC's electronic filing +system, and the agency's Division of Economic and Risk Analysis republishes those filings as +structured, tab separated data sets in three month batches. The history is built from those data +sets. The daily job reads each day's filings from EDGAR itself, the information tables the data sets +are extracted from, because a batch arrives up to three months after its first filing: compared +line by line, 48 of 48 filings carry the same lines in both, and on six sample days EDGAR's daily +index lists exactly the filings the data set holds. The data is public domain, needs no account and +no API key, and the only access requirement is the descriptive User-Agent header that SEC policy +asks of all automated readers. + +## Getting Started + +```python +self._symbol = self.add_equity("AAPL", Resolution.DAILY).symbol +self._holdings_symbol = self.add_data(SEC13F, self._symbol).symbol +``` +```csharp +_symbol = AddEquity("AAPL", Resolution.Daily).Symbol; +_holdingsSymbol = AddData(_symbol).Symbol; +``` + +## Data Summary + +The following table describes the dataset properties: + +| Property | Value | +| --- | --- | +| Start Date | May 2013 | +| Asset Coverage | 6,616 US Equities | +| Data Density | Sparse | +| Resolution | Daily\* | +| Timezone | America/New_York | + +\* Positions are reported quarterly, but the managers of one quarter file across roughly fifty +different days and several quarters are live at once, so publication is close to continuous rather +than quarterly. Measured on the processed files over the twelve months to May 2026, a security +carries a release on a median of 115 days (p10 29, p90 170); a widely held name such as AAPL +carries one on 234 of the roughly 250 business days. + +The history begins on 2013-05-20, the first filing date in the SEC structured data set, whose first +archive covers the second quarter of 2013. Anything earlier exists only as raw filings in the EDGAR +full index and is not part of this dataset. + +Each reading in a data point's `Holdings` carries the following fields, all summed across the +managers that reported the security for that quarter: + +| Property | Meaning | +| --- | --- | +| `PeriodEnd` | End of the quarter the positions are reported for, the SEC PERIODOFREPORT | +| `Quarter` | The same quarter as a sortable label such as `2020Q2`, derived from `PeriodEnd` | +| `Holders` | Number of distinct managers, counted by filer CIK, that have reported the security | +| `Shares` | Shares held, summed over lines with share type SH and no option flag | +| `HoldingValue` | Market value of those same lines as reported by the managers; the `MostReported` reading's is the data point's `Value` | +| `CallShares` | Shares underlying reported call positions | +| `PutShares` | Shares underlying reported put positions | +| `PrincipalValue` | Principal amount of debt instruments reported for the security, share type PRN | +| `VotingSole` | Shares over which the reporting managers hold sole voting authority | +| `VotingShared` | Shares over which the reporting managers share voting authority | +| `ConfidentialOmitted` | True when at least one contributing filing withheld positions under confidential treatment | + +Option lines and debt principal are deliberately kept out of `Shares`. One quarter of the source +carries 61,400 call lines, 57,443 put lines and 15,698 PRN lines, and folding any of them into a +share count would misstate the position, so they are carried as their own fields instead. For the +same reason the reported value column is only meaningful under the same filter: summed raw, with +option and PRN lines left in, it totals 70.1 trillion dollars for a single quarter. + +`ConfidentialOmitted` is a point-in-time feature rather than a data quality flag. A manager can ask +the SEC to withhold specific positions temporarily, so a filing marked this way is incomplete by +design and the withheld positions appear in a later release. + +### How the aggregation was validated + +Every reported line is summed. Lines are not deduplicated by investment discretion and lines that +name another manager are not dropped. That looks like double counting, because 44.3 percent of +holding lines name another manager and those lines carry 54.9 percent of the reported value, so the +rule was measured rather than assumed. The three largest holdings of the 2026-03-31 period were +summed against shares outstanding, counting only share type SH with an empty option flag: + +| Security | Raw sum | Sole discretion only | Lines naming no other manager | +| --- | --- | --- | --- | +| MSFT | 71.6% | 25.6% | 29.5% | +| AAPL | 63.1% | 22.6% | 26.4% | +| NVDA | 66.0% | 22.9% | 26.6% | + +Published institutional ownership depends on the provider: in 2026 it runs from about 53 to 74 +percent for AAPL, 54 to 70 for NVDA and 72 to 82 for MSFT. The raw sum lands inside those spreads, +or for MSFT just under them, while both of the obvious defences land far below every published +figure, because holdings reported under +defined discretion are legitimate holdings and dropping them throws away most of the institutional +base. `Holders`, being a count of distinct filer CIKs, is immune to the question either way and is +the safer headline field. + +Amendments are the one place where lines are not simply summed. Most amendments restate the whole +report, and adding a restatement on top of the report it replaces counted the same position twice: +for Apple's March 2026 quarter that was 993,574,019 shares, 9.6 percent of the total. An amendment's +lines therefore count only for a security the manager had not reported for the quarter yet, which is +what an amendment adding new holdings carries. The rule only ever adds: a restatement that lowers or +removes a position leaves the original figure standing, and an amendment adding holdings to a security +the manager already reported for the quarter is not counted. Against replacing each manager's position +with its restatement, the difference on the three largest holdings of the March 2026 quarter is under +0.01 percent. + +That count is taken once per manager: a manager that reports the security on several filings, under +several of the issuer's CUSIPs, or names it for the first time in an amendment counts once for the +quarter. Checked against the source tables, Apple's March 2026 quarter reads 6,041 holders where the +SEC data holds 6,040 to 6,043 distinct managers, depending on which share classes count as Apple. + +### Coverage is partial by design + +Holdings are keyed by CUSIP in the source and resolved to a LEAN `Symbol` before publication, so no +identifier from the source is shipped. The resolution runs in three steps, each picking up what the +one before it could not reach: + +1. The CUSIP itself, looked up in LEAN's security database. On its own it accounts for 76.1 percent + of the reported value. The database repeats some identifiers across the listings one company has + had, such as the old and the new Alcoa, so every row carrying the CUSIP is tried and the one + trading under its own ticker on the filing date is kept. +2. The US ISIN, built arithmetically from the same CUSIP. It reaches issuers whose CUSIP column in + that database is blank and lifts the two steps together to 88.9 percent of value. +3. The ticker the SEC's own Form N-PORT filings report for the CUSIP, taken back to a `Symbol` + through the map files. This step needs no security database at all and it is what reaches the + foreign domiciled issuers whose identifier is really a CINS, for which a constructed US ISIN is + wrong by construction. Alphabet is one of them, which is why `GOOGL` appears in the demo + algorithms. With this step the chain covers 98.2 percent of reported value on the most recent + window. A fund's reported ticker is free text, so this step keeps a match only when the reported + prices do not say otherwise. A CUSIP whose issue number carries letters, as debt does, is kept + only when its reported prices are the security's own quarter-end close, in dollars or in + thousands, and any other group is dropped when three or more of its prices are not. That keeps + Etsy's convertible notes out of Etsy's share count, and Centerra Gold, Enerflex, B2Gold and DeFi + Technologies, whose Toronto or fund-reported tickers are CG, EFX, BTO and DEFI, off Carlyle, + Equifax, a John Hancock fund and the Hashdex DEFI ETF. A group that cannot be checked, such as a + day of option positions only or one or two lines that are usually a single manager's slip, is + kept. + +The real limit is the third step's own history: N-PORT begins in late 2019, so a security that +stopped trading before then is not reachable through it at any depth and can only be resolved if +the first two steps already found it. This dataset therefore covers most of the reported dollars +but not every reported name, and it says so rather than implying full coverage. + +Raw sums only. Quarter over quarter change, percentage of shares outstanding, concentration ratios +and new or closed position counts are all derivable from what is shipped, so they are left to the +algorithm, which is also the only place that knows the window and the ranking the strategy needs. + +## Example Applications + +The SEC Form 13F Institutional Holdings dataset lets you see what large managers actually hold and +trade against how crowded a name is. Examples include the following strategies: + +- Screening for crowding by ranking the universe on the number of holders, and for de-crowding by + taking the securities whose holder count fell hardest against the previous quarter. +- Building an ownership change momentum signal from the quarter over quarter move in reported + shares, and going long the names institutions are accumulating. +- Filtering an existing universe by institutional breadth, requiring a minimum holder count before + a security is tradeable so that thinly followed names never enter the portfolio. +- Avoiding names where institutional ownership is collapsing, using a falling holder count and a + falling share total together as an exit condition. +- Reading the reported put and call share totals alongside the share position to see whether + managers are hedging a name rather than simply owning it. + +## Meta + +| Field | Value | +| --- | --- | +| name | SEC Form 13F Institutional Holdings | +| url | sec-form-13f-institutional-holdings | +| vendorName | U.S. Securities and Exchange Commission | +| website | https://www.sec.gov | +| history | May 2013 | +| reach | 6,616 US Equities | +| shortDescription | Institutional ownership per US Equity aggregated from every Form 13F filing, published quarterly by the SEC | +| priceCTA | Free in Cloud | +| delivery | cloud only | + +Tags: Financial Market Data + +Licensing card: + +```html +

Free access to SEC Form 13F Institutional Holdings in QuantConnect Cloud for use in backtesting or live trading.

+
    +
  • Quarterly institutional ownership across 6,616 US Equities, delivered on the filing date
    • +
  • Holder counts, share and value totals, option and debt lines, and voting authority, per security and as a universe
    • +
  • Curated, clean data
    • +
+``` diff --git a/listing-documentation-13f.md b/listing-documentation-13f.md new file mode 100644 index 0000000..63cfbe7 --- /dev/null +++ b/listing-documentation-13f.md @@ -0,0 +1,254 @@ +## Requesting Data + +To add SEC Form 13F Institutional Holdings data to your algorithm, call the **AddData** method with +the `Symbol` of an Equity you already subscribe to. Save a reference to the dataset **Symbol** so you +can access the data later in your algorithm. + +```python +class SEC13FDataAlgorithm(QCAlgorithm): + + def initialize(self) -> None: + # Coverage runs 2013-05-20 to 2026-05-29 + self.set_start_date(2020, 10, 1) + self.set_end_date(2020, 12, 31) + self.set_cash(100000) + + self._symbol = self.add_equity("AAPL", Resolution.DAILY).symbol + self._dataset_symbol = self.add_data(SEC13F, self._symbol).symbol +``` +```csharp +public class SEC13FDataAlgorithm : QCAlgorithm +{ + private Symbol _symbol, _datasetSymbol; + + public override void Initialize() + { + // Coverage runs 2013-05-20 to 2026-05-29 + SetStartDate(2020, 10, 1); + SetEndDate(2020, 12, 31); + SetCash(100000); + + _symbol = AddEquity("AAPL", Resolution.Daily).Symbol; + _datasetSymbol = AddData(_symbol).Symbol; + } +} +``` + +## Accessing Data + +To get the current SEC Form 13F Institutional Holdings data, index the current [Slice](https://www.quantconnect.com/docs/v2/writing-algorithms/key-concepts/time-modeling/timeslices) +with the dataset **Symbol**. **Slice** objects deliver unique events to your algorithm as they +happen, but the **Slice** may not contain data for your dataset at every time step. Positions are +reported quarterly, but the managers of one quarter file across roughly fifty different days, so +publication is close to continuous: measured on the processed files a security carries a release on +a median of 111 days a year, and a widely held name on 234 of the roughly 250 business days. Check +that the **Slice** contains the data you want before you index it. + +```python +def on_data(self, slice: Slice) -> None: + if self._dataset_symbol in slice: + data_point = slice[self._dataset_symbol] + holders = data_point.most_reported.holders + shares = data_point.most_reported.shares +``` +```csharp +public override void OnData(Slice slice) +{ + if (slice.ContainsKey(_datasetSymbol)) + { + var dataPoint = slice[_datasetSymbol]; + var holders = dataPoint.MostReported.Holders; + var shares = dataPoint.MostReported.Shares; + } +} +``` + +To iterate through all of the dataset objects in the current **Slice**, call the **Get** method. A +filing deadline puts thousands of securities into the same day, so this is the usual shape when you +subscribe to more than one name. + +```python +def on_data(self, slice: Slice) -> None: + for dataset_symbol, data_point in slice.get(SEC13F).items(): + holders = data_point.most_reported.holders +``` +```csharp +public override void OnData(Slice slice) +{ + foreach (var kvp in slice.Get()) + { + var datasetSymbol = kvp.Key; + var dataPoint = kvp.Value; + var holders = dataPoint.MostReported.Holders; + } +} +``` + +A data point's `Time` and `EndTime` are both the moment it was published, 03:00 ET on the day after +the filing date, and the quarters the numbers describe are carried inside the point. EDGAR lists a +day's filings at about 22:05 ET and the daily job reads them after midnight, so this is the earliest +a live algorithm can have them, and history uses the same stamp. Your algorithm therefore reads a +position only once the dataset could have delivered it. +The gap between the two is not a constant and cannot be derived: measured across 11,761 filings in +one window it runs minimum 0 days, p10 16, median 42, p90 48, maximum 6,596, and 10.4 percent of +filings arrive later than the 45 day deadline. A point that arrives today therefore describes a +quarter that ended typically 45 to 135 days ago, with late amendments arriving years later, and a +single day of filings carries several different reported quarters. Read `PeriodEnd` whenever you +need the quarter a reading describes rather than the day it arrived. + +That last point is why a data point is not a single reading. One release restates every quarter of +the security whose numbers moved that day, and LEAN delivers one data point per security per +timestamp, so the point carries them all in `Holdings`: one `SEC13FHolding` per reported quarter, +oldest first and never empty. Seven percent of the points in the history carry more than one, up to +75. Each holding also exposes `Quarter`, the reported quarter as a sortable label such as `2020Q2`, +derived from `PeriodEnd` so the two can never disagree. + +```python +def on_data(self, slice: Slice) -> None: + if self._dataset_symbol in slice: + data_point = slice[self._dataset_symbol] + for holding in data_point.holdings: + self.log(f"{holding.quarter}: {holding.holders} holders, {holding.shares} shares") +``` +```csharp +public override void OnData(Slice slice) +{ + if (slice.ContainsKey(_datasetSymbol)) + { + SEC13F dataPoint = slice[_datasetSymbol]; + foreach (var holding in dataPoint.Holdings) + { + Log($"{holding.Quarter}: {holding.Holders} holders, {holding.Shares} shares"); + } + } +} +``` + +Two shortcuts pick a single reading out of the point. `MostReported` is the quarter the most +managers have reported and `Latest` is the newest quarter. They differ during a quarter handover, +which lasts about six weeks: the new quarter has a handful of managers in it while the quarter it +replaces has thousands, so `Latest` is the thin one and `MostReported` is the finished one. Rank a +cross-section on `MostReported`, which compares complete readings; take `Latest` only when you +deliberately want the freshest read and have accounted for how incomplete it is. The point's `Value` +is `MostReported.HoldingValue`. + +Every value is cumulative for its `PeriodEnd`, counting every filing for that quarter that was +public by the point's timestamp. You do not have to accumulate anything yourself, and you should +not: adding two points of the same quarter together would double count. + +Every numeric field is declared nullable, but every shipped row populates all of them: a quarter +that carries share lines and no option lines reports zero for the option totals rather than leaving +them empty. Zero means the managers reported nothing of that kind, which is a reading and not a gap. +The nullability is there so the type can express a gap if the source ever produces one, so the +defensive check costs nothing, but do not expect `null` in the published history. + +```python +holders = data_point.most_reported.holders or 0 +shares_per_holder = data_point.most_reported.shares / holders if holders else 0 +``` +```csharp +var mostReported = dataPoint.MostReported; +var holders = mostReported.Holders ?? 0m; +var sharesPerHolder = holders == 0m ? 0m : mostReported.Shares.GetValueOrDefault() / holders; +``` + +`ConfidentialOmitted` is true when at least one of the filings behind that quarter withheld +positions under confidential treatment. Such a filing is incomplete by design and the withheld +positions surface in a later release, so treat a flagged reading as a floor on the reported position +rather than as the full picture. + +## Historical Data + +To get historical SEC Form 13F Institutional Holdings data, call the **History** method with the +dataset **Symbol**. If there is no data in the period you request, the history result is empty. + +```python +# DataFrame, one row per release +history_df = self.history(self._dataset_symbol, timedelta(days=730), Resolution.DAILY) + +# DataFrame, one row per reported quarter +history_flat_df = self.history(SEC13F, self._dataset_symbol, timedelta(days=730), Resolution.DAILY, flatten=True) + +# Dataset objects, one per release +history_bars = self.history[SEC13F](self._dataset_symbol, timedelta(days=730), Resolution.DAILY) +``` +```csharp +var history = History(_datasetSymbol, TimeSpan.FromDays(730), Resolution.Daily); +``` + +The typed history and the C# history give you the points themselves, one per release, each carrying +its own `Holdings`. A release that restated several quarters is one object, so read `Holdings` to +see them rather than expecting one object per quarter. + +In Python the DataFrame has two shapes, and the difference matters. Because a point enumerates its +quarters, passing `flatten=True` expands it into **one row per reported quarter**, which is the form +to use whenever you want to line quarters up or build a per-quarter series. Without it you get **one +row per release**, whose columns are the fields of that release's `MostReported` reading plus a +`holdings` column holding the full list, so a release that restated ten quarters still occupies a +single row and nine of its readings are only reachable through that column. + +Ask for a time span rather than a bar count. A bar count is read in daily bars and the dataset +publishes on filing dates only, so what comes back depends on how widely the security is held +rather than on the count you asked for. Measured on the processed files, AAPL carries a release on +96 of the last 100 business days and on 235 of the last 365 calendar days, while a thinly held name +carries one on closer to 40 days a year. A time span is the safer form whenever you need a known +number of quarters of history. + +For more information about historical data, see [History Requests](https://www.quantconnect.com/docs/v2/writing-algorithms/historical-data/history-requests). + +## Universe Selection + +To select a dynamic universe of US Equities based on SEC Form 13F Institutional Holdings data, call +the **AddUniverse** method with the **SEC13FUniverse** class and a selection function. The universe +file is keyed by release date and a single release date carries filings for several reported +quarters, but each security still arrives as a single `SEC13F` record with its quarters travelling +together, so your selection function sees one record per security. At most two quarters are live +per security there, the one still filling in and the one it replaces, and `MostReported` is the +complete one of the two. + +```python +def initialize(self) -> None: + self.universe_settings.resolution = Resolution.DAILY + self.add_universe(SEC13FUniverse, self._select) + +def _select(self, data: List[SEC13F]) -> List[Symbol]: + # Select the most widely held securities of the release + ranked = [datum for datum in data if datum.most_reported.holders is not None] + selected = sorted(ranked, + key=lambda datum: (-datum.most_reported.holders, datum.symbol.value))[:20] + return [datum.symbol for datum in selected] +``` +```csharp +public override void Initialize() +{ + UniverseSettings.Resolution = Resolution.Daily; + + AddUniverse(data => + { + // Select the most widely held securities of the release + return (from SEC13F datum in data + let quarter = datum.MostReported + where quarter.Holders.HasValue + orderby quarter.Holders.Value descending, datum.Symbol.Value + select datum.Symbol).Take(20); + }); +} +``` + +For more information about dynamic universes, see [Universes](https://www.quantconnect.com/docs/v2/writing-algorithms/universes/key-concepts). + +## Remove Subscriptions + +To remove your subscription to SEC Form 13F Institutional Holdings data, call the **RemoveSecurity** +method. + +```python +self.remove_security(self._dataset_symbol) +``` +```csharp +RemoveSecurity(_datasetSymbol); +``` + +If you subscribe to SEC Form 13F Institutional Holdings data for assets in a dynamic universe, +remove the dataset subscription when the asset leaves your universe. To view a common design +pattern, see [Track Security Changes](https://www.quantconnect.com/docs/v2/writing-algorithms/algorithm-framework/alpha/key-concepts#05-Track-Security-Changes). diff --git a/output/alternative/sec/13f/aapl.csv b/output/alternative/sec/13f/aapl.csv new file mode 100644 index 0000000..c173dcb --- /dev/null +++ b/output/alternative/sec/13f/aapl.csv @@ -0,0 +1,57 @@ +20201002 03:00,1,20200930,11,4091893,473319000,0,0,0,3104369,320,0 +20201003 03:00,2,20161231,2470,3154049434,364989445103,122931844,128198549,187826,1866917682,262289718,1,20200930,18,4958860,573723000,0,0,0,3878678,320,1 +20201006 03:00,1,20200930,35,9689407,1123082000,0,0,0,8174627,1880,1 +20201007 03:00,3,20170331,2510,3245250321,465567509468,106251171,91826772,60698,1935338352,264912919,1,20170930,2510,3119974969,479855370750,97802150,97142251,53378,1870483340,273454202,1,20200930,69,14907129,1750280233,0,160,0,11581134,3768,1 +20201008 03:00,4,20170630,2507,3177633184,456989193947,95789819,90784993,46317,1915761746,248685887,1,20200331,3278,2686774956,653580418467,67139739,72200152,27882,1652064582,169398880,1,20200630,3323,2593821891,931028543399.059,62056050,60809616,9515,1617836453,171210197,1,20200930,95,17230133,2018852233,0,160,0,13057035,17606,1 +20201009 03:00,1,20200930,137,25809440,3012204066,0,160,0,19079235,143648,1 +20201010 03:00,1,20200930,176,39295067,4573620066,400,560,0,28711663,172403,1 +20201014 03:00,5,20171231,2780,3117342694,527451577682,78072145,109735771,53504,1907072163,268779226,1,20180331,2765,3025601683,502853506312,63212872,77314656,149497,2011700805,273784901,1,20180930,2821,2858122284,642001452732,66046157,78957296,83329,1867929981,250674626,1,20200331,3279,2686774956,653580418467,67139739,72200152,61907,1652064582,169398880,1,20200930,292,56867940,6605513066,13200,560,0,38735805,367467,1 +20201015 03:00,2,20180630,2768,2931487749,538850232067,65194509,75334865,82659,1808375415,248687057,1,20200930,361,75989294,8810423071,14800,3360,0,46762952,5814007,1 +20201016 03:00,1,20200930,412,87565867,10152043071,14800,3360,0,55685812,6477369,1 +20201017 03:00,2,20191231,3303,2694891187,775312763820,93052535,97667837,120055,1732100886,176233512,1,20200930,474,97168833,11361440573,14800,162960,0,61340763,6511738,1 +20201020 03:00,1,20200930,562,120716866,14088930573,157063,242434,0,80266579,6607992,1 +20201021 03:00,2,20200630,3324,2593823376,931029085399.059,62056050,60809616,9515,1617837938,171210197,1,20200930,639,157484024,18347940573,161063,642182,0,112233684,6607992,1 +20201022 03:00,1,20200930,705,181930015,21141618592,165063,784882,0,134381394,6625583,1 +20201023 03:00,2,20200630,3325,2593825742,931029948399.059,62056050,60809616,9515,1617839777,171210197,1,20200930,766,196351669,22803325261,165063,884882,0,144301015,6625583,1 +20201024 03:00,2,20181231,3013,2990291120,446387632150,68155380,82481056,67337,2011139353,206407855,1,20200930,848,208002125,24154113644,170063,1087798,0,151474348,6789146,1 +20201027 03:00,1,20200930,940,237603684,27585584434,419488,1414923,179897,176260922,7186046,1 +20201028 03:00,2,20200630,3326,2593827856,931030719399.059,62056050,60809616,9515,1617839777,171210197,1,20200930,1024,262000144,30450716434,1198888,1832723,179897,195580088,7230642,1 +20201029 03:00,1,20200930,1102,281292371,32685377434,1423688,2266523,179897,206129245,13478469,1 +20201030 03:00,1,20200930,1187,335219945,38915282434,1435688,2298123,179897,240020571,14222551,1 +20201031 03:00,1,20200930,1279,502482474,58326680208,1484162,2441468,179897,381248549,32227185,1 +20201103 03:00,7,20190331,2994,2751578536,521068876513,82881196,84287410,65346,1857651654,204784348,1,20190630,2982,2708584347,534608016316,62355086,82092772,60882,1750317788,209585535,1,20190930,2991,2629702038,588510386749,77825894,82911630,829,2446394107,205021488,1,20191231,3304,2694893252,775313369820,93052535,97667837,120055,1732100886,176233512,1,20200331,3280,2686777016,653580942467,67139739,72200152,61907,1652064582,169398880,1,20200630,3327,2593836160,931097500399.059,62128750,60820416,9515,1617848081,171210197,1,20200930,1371,542538359,63024209872,1880781,2441493,323322,405686028,32906196,1 +20201104 03:00,1,20200930,1495,583656879,67796108182,1897781,2487993,323322,429468870,35483834,1 +20201105 03:00,1,20200930,1585,721657922,83755142343,2305881,2517493,323322,524962745,41161027,1 +20201106 03:00,2,20200630,3329,2593846426,931101245399.059,62128750,60820416,9515,1617853675,171210197,1,20200930,1690,931150080,108071204326,4821281,5704841,323322,695459972,41733069,1 +20201107 03:00,2,20200630,3330,2593998975,931156895399.059,62128750,60820416,9515,1618006224,171210197,1,20200930,1794,2186512123,250840711020,6839831,6549741,323322,1751081437,62844626,1 +20201110 03:00,1,20200930,1942,2413493728,277125289352,7107031,6938841,323322,1915747491,67944880,1 +20201111 03:00,2,20181231,3014,2990291120,446387632150,68155380,82481056,116813,2011139353,206407855,1,20200930,2105,3238509099,372680064632,7147831,6956785,323322,2623557761,84837625,1 +20201113 03:00,7,20181231,3015,2990295483,446388320150,68155380,82481056,116813,2011139414,206407855,1,20190331,2995,2751583407,521069801513,82881196,84287410,65346,1857651756,204784348,1,20190630,2983,2708589454,534609027316,62355086,82092772,60882,1750317920,209585535,1,20190930,2992,2629707280,588511560749,77825894,82911630,829,2446394260,205021488,1,20191231,3305,2694898192,775314820820,93052535,97667837,120055,1732101039,176233512,1,20200331,3281,2686782755,653582401467,67139739,72200152,61907,1652064735,169398880,1,20200930,2529,4422426158,508083527582,39479891,49472685,323322,3551215918,131071704,1 +20201114 03:00,3,20190930,2993,2677066095,599118514749,77825894,82911630,829,2493753075,205021488,1,20191231,3306,2694899752,775315278820,93052535,97667837,120055,1732101039,176233512,1,20200930,2945,6173804333,710865538773,93151391,132427133,352074,4608041427,181671946,1 +20201117 03:00,3,20200331,3282,2686784031,653582725467,67139739,72200152,61907,1652064735,169398880,1,20200630,3331,2594011591,931161497399.059,62128750,60820416,9515,1618006224,171210197,1,20200930,3274,9663010703,1114962933259,265228991,291501233,649898,6140539968,640717603,1 +20201118 03:00,1,20200930,3303,9703658181,1119672532259,325797091,312145433,649898,6179411006,641098575,1 +20201119 03:00,3,20191231,3307,2694900427,775315476820,93052535,97667837,120055,1732101039,176233512,1,20200630,3333,2594024286,931171216399.059,62128750,60820416,9515,1618018794,171210197,1,20200930,3314,9704601839,1119781812259,325797091,312145433,649898,6180003180,641098575,1 +20201120 03:00,1,20200930,3326,9706041912,1119948586927,325799391,312302433,649898,6180139875,641098575,1 +20201121 03:00,1,20200930,3327,9706082057,1119952956927,325799391,312302433,649898,6180139875,641098575,1 +20201124 03:00,3,20200331,3283,2686786708,653583406467,67139739,72200152,61907,1652064735,169398880,1,20200630,3334,2594027210,931172283399.059,62128750,60820416,9515,1618018794,171210197,1,20200930,3328,9706153389,1119961216927,325799391,312302433,649898,6180139875,641098575,1 +20201125 03:00,2,20200331,3284,2686850118,653599564467,67725239,72488952,61907,1652128145,169398880,1,20200930,3329,9706218273,1119968730927,325799391,312302433,649898,6180202939,641098575,1 +20201126 03:00,1,20200930,3331,9707264545,1120089899927,325799391,312302433,649898,6181202491,641098575,1 +20201128 03:00,1,20200930,3333,9707509622,1120118286927,325799391,312302433,649898,6181202491,641098575,1 +20201201 03:00,3,20200331,3285,2686873676,653605555467,67725239,72488952,61907,1652128145,169398880,1,20200630,3335,2594051685,931181212399.059,62128750,60820416,9515,1618018794,171210197,1,20200930,3338,9708708022,1120257221927,325799391,312302433,649898,6181711307,641098575,1 +20201202 03:00,1,20200930,3341,9709977689,1120404255927,325799391,312302433,649898,6182945414,641098575,1 +20201203 03:00,1,20200930,3342,9712809937,1120732258927,325799391,312302433,649898,6185777662,641098575,1 +20201204 03:00,2,20191231,3308,2694918932,775320910820,93052535,97667837,120055,1732101039,176233512,1,20200930,3346,9713872709,1120855337927,325799391,312302433,649898,6186418757,641098575,1 +20201208 03:00,1,20200930,3347,9713920533,1120860925927,325799391,312302433,649898,6186422616,641098575,1 +20201209 03:00,1,20200930,3348,9713973161,1120867020927,325799391,312302433,649898,6186475244,641098575,1 +20201210 03:00,1,20200930,3349,9713995282,1120869582927,325799391,312302433,649898,6186475244,641098575,1 +20201211 03:00,1,20200930,3352,9715834519,1121086209927,325799391,312302433,649898,6188314481,641098575,1 +20201212 03:00,1,20200930,3355,9716666988,1121182617927,325799391,312302433,649898,6188987784,641098575,1 +20201215 03:00,1,20200930,3358,9727385023,1122423872927,325799391,312302433,649898,6198837491,641098575,1 +20201216 03:00,2,20200630,3336,2594062591,931185191399.059,62128750,60820416,9515,1618029700,171210197,1,20200930,3359,9727425775,1122428592927,325799391,312302433,649898,6198878243,641098575,1 +20201217 03:00,1,20200630,3337,2594162727,931221721399.059,62128750,60820416,9515,1618129836,171210197,1 +20201219 03:00,4,20141231,2305,3487297320,384971479822,142776039,217908062,114160,2360116907,209037629,1,20150331,2342,4155884055,427300763033,109850085,124860328,800,2781576756,245547580,1,20200331,3286,2686901422,653612610467,67725239,72488952,61907,1652155891,169398880,1,20200930,3361,9728069924,1122505176927,325799391,312312833,649898,6198938718,641098575,1 +20201222 03:00,1,20200930,3362,9729976277,1122725951927,325799391,312312833,649898,6200765013,641098575,1 +20201223 03:00,4,20150930,2325,3287887975,362135660170,125455478,147845401,13945,1883236364,334996445,1,20151231,2411,3261283333,341312331356,89945266,128516536,1871990,1869608294,339758054,1,20160331,2410,3344946831,346619038427,129825964,81692443,34223,2014911800,276218610,1,20200930,3363,9729982640,1122726688927,325799391,312312833,649898,6200771376,641098575,1 +20201224 03:00,12,20170331,2511,3245251907,465567737468,106251171,91826772,60698,1935338352,264912919,1,20170630,2508,3177634771,456989422947,95789819,90784993,46317,1915761746,248685887,1,20170930,2511,3119976556,479855615750,97802150,97142251,53378,1870483340,273454202,1,20171231,2781,3117344282,527451846682,78072145,109735771,53504,1907072163,268779226,1,20180930,2822,2858123540,642001731732,66046157,78957296,83329,1867929981,250674626,1,20190331,2996,2751584956,521070095513,82881196,84287410,65346,1857651756,204784348,1,20190630,2984,2708591030,534609339316,62355086,82092772,60882,1750317920,209585535,1,20190930,2994,2677067685,599118870749,77825894,82911630,829,2493753075,205021488,1,20191231,3309,2694920605,775321401820,93052535,97667837,120055,1732101039,176233512,1,20200331,3287,2686903146,653613048467,67725239,72488952,61907,1652155891,169398880,1,20200630,3339,2594191093,931232068399.059,62128750,60820416,9515,1618156376,171210197,1,20200930,3365,9729994429,1122728053927,325799391,312312833,649898,6200771376,641098575,1 +20201230 03:00,1,20200930,3366,9730059060,1122735537927,325799391,312312833,649898,6200836007,641098575,1 +20201231 03:00,4,20190331,2997,2751584956,521070095513,82881196,84287410,126555,1857651756,204784348,1,20190630,2985,2708591030,534609339316,62355086,82092772,113036,1750317920,209585535,1,20190930,2995,2677067685,599118870749,77825894,82911630,52456,2493753075,205021488,1,20200630,3340,2594191093,931232068399.059,62128750,60820416,41673,1618156376,171210197,1 diff --git a/output/alternative/sec/13f/aig.csv b/output/alternative/sec/13f/aig.csv new file mode 100644 index 0000000..b228813 --- /dev/null +++ b/output/alternative/sec/13f/aig.csv @@ -0,0 +1,43 @@ +20201002 03:00,1,20200930,1,125,3000,0,0,0,125,0,0 +20201003 03:00,1,20161231,938,843278852,55019352598,11826556,11917146,11142,537752506,94271787,1 +20201006 03:00,1,20200930,2,199450,5490000,0,0,0,199450,0,0 +20201007 03:00,2,20170331,887,801391250,49999759509,10652469,8210240,170,510366070,94104214,1,20200930,8,253571,6979000,0,0,0,212840,0,0 +20201008 03:00,1,20200930,10,253584,6979000,0,0,0,212853,0,0 +20201009 03:00,1,20200930,17,814025,22408000,0,0,0,761142,4064,0 +20201010 03:00,1,20200930,22,1863033,51286000,0,0,0,1740936,4064,0 +20201014 03:00,1,20200930,33,1874445,51601000,532200,1108100,0,1750192,4064,0 +20201015 03:00,1,20200930,42,2237144,61587663,532200,1108100,0,1966135,106315,1 +20201016 03:00,1,20200930,50,2352876,64773663,532200,1108100,0,2026286,106315,1 +20201017 03:00,1,20200930,57,2687186,73976663,532200,1108100,0,2240609,106315,1 +20201020 03:00,1,20200930,74,3527041,97098663,532200,1109654,0,2675574,106565,1 +20201021 03:00,1,20200930,90,4268692,117514663,532200,1131341,0,3382047,106565,1 +20201022 03:00,1,20200930,101,5252476,144597663,532200,1131341,0,4364869,106565,1 +20201023 03:00,1,20200930,109,5711252,157228663,532200,1131341,0,4638827,106565,1 +20201024 03:00,2,20181231,789,806162936,31254423268,13028227,9097147,0,496362870,85750182,1,20200930,117,6280671,172902663,532200,1131341,0,5144698,106565,1 +20201027 03:00,1,20200930,131,8072630,222234939,532200,1131341,0,6922800,106565,1 +20201028 03:00,1,20200930,142,9096963,252398939,569400,1153241,0,7773972,106565,1 +20201029 03:00,1,20200930,149,9342804,259167939,569400,1153241,0,7806941,316630,1 +20201030 03:00,1,20200930,160,10112339,280305939,569400,1153241,0,8330129,339372,1 +20201031 03:00,1,20200930,181,36409265,1005440691,575012,1164438,0,31331321,1715278,1 +20201103 03:00,1,20200930,193,36786721,1015922691,575012,1164438,0,31678026,1719343,1 +20201104 03:00,1,20200930,209,38162470,1053798691,575012,1164438,0,33021993,1729142,1 +20201105 03:00,1,20200930,222,41869159,1141833691,575012,1164438,0,36403992,1888196,1 +20201106 03:00,1,20200930,243,53232267,1454764691,575012,1164438,0,44927033,3244651,1 +20201107 03:00,1,20200930,271,136656426,3719285691,575038,1164438,0,120111870,4074531,1 +20201110 03:00,1,20200930,307,188850591,5225975691,663838,1214438,0,163476393,4145177,1 +20201111 03:00,1,20200930,331,231804858,6408482425,663838,1214438,0,201654104,4691194,1 +20201113 03:00,1,20200930,432,322527890,8905164425,1033738,3324738,0,276178379,7458154,1 +20201114 03:00,2,20190930,812,803067620,44652139958,10780745,7397202,0,487633538,78564019,1,20200930,542,490285207,13523471982,5759568,8947138,0,397453915,24752304,1 +20201117 03:00,1,20200930,628,762838163,20974206289,10173268,17334438,0,491284427,72384594,1 +20201118 03:00,1,20200930,638,765566029,21049454289,10563968,18398838,0,493826192,72508312,1 +20201119 03:00,2,20200331,699,776420478,18803503367,7890971,9857904,526,489449777,95959863,1,20200630,677,768528537,23984025747,9363225,12100518,0,492413291,74555899,1 +20201120 03:00,1,20200930,643,765648880,21051736065,10563968,18398838,0,493827192,72508312,1 +20201121 03:00,1,20200630,678,768536674,23984279467,9363225,12100518,0,492413291,74555899,1 +20201125 03:00,1,20200331,700,776526794,18806236367,7961171,9965104,526,489556093,95959863,1 +20201201 03:00,1,20200930,644,765649431,21051758065,10563968,18398838,0,493827743,72508312,1 +20201203 03:00,1,20200930,645,765726206,21053872065,10563968,18398838,0,493904518,72508312,1 +20201204 03:00,1,20200930,646,765885617,21058261065,10563968,18398838,0,494063929,72508312,1 +20201209 03:00,1,20200930,647,766231601,21067786065,10563968,18398838,0,494409913,72508312,1 +20201211 03:00,1,20200930,649,766240363,21068027065,10563968,18398838,0,494418675,72508312,1 +20201215 03:00,1,20200930,650,766580554,21077392065,10563968,18398838,0,494758866,72508312,1 +20201222 03:00,1,20200930,651,766586617,21077559065,10563968,18398838,0,494764929,72508312,1 diff --git a/output/alternative/sec/13f/bac.csv b/output/alternative/sec/13f/bac.csv new file mode 100644 index 0000000..c707dd1 --- /dev/null +++ b/output/alternative/sec/13f/bac.csv @@ -0,0 +1,51 @@ +20201002 03:00,1,20200930,6,756603,18260000,0,0,0,571542,0,0 +20201003 03:00,2,20161231,1724,6733197763,148852974992,252736054,210682836,318144,4131490074,520042981,1,20200930,9,1029191,24827000,0,0,0,830166,0,1 +20201006 03:00,1,20200930,18,2543323,61347000,0,0,0,2338057,0,1 +20201007 03:00,3,20170331,1760,6656954459,157303014331,167772367,172086782,290242,4101363920,529721802,1,20170930,1781,7252102099,183941616104,152368485,196951498,254199,4732200345,539097945,1,20200930,37,5958568,143562578,0,0,0,4734425,365,1 +20201008 03:00,4,20170630,1765,6613226066,160442190364,181337055,185514357,254177,4101486785,502755659,1,20200331,2058,5990612359,127408454692,107966213,136917841,53478,4078609217,539385431,1,20200630,2092,5920326659,141115058534.821,107384568,157707032,187805,4003759522,536801725,1,20200930,49,8431054,203698578,0,0,0,7070918,6643,1 +20201009 03:00,1,20200930,74,12398039,299330382,0,0,0,9946257,44175,1 +20201010 03:00,1,20200930,95,17885579,436429382,0,0,0,14661589,52000,1 +20201014 03:00,5,20171231,1979,7161176271,211267454603,170798978,209272097,264047,4623729080,516409779,1,20180331,2020,6984948151,209177987718,135783564,172759094,271058,4781024583,542157731,1,20180930,2010,6715282138,197706265111.460,112692347,141801620,414691,4580129137,527009613,1,20200331,2059,5990612359,127408454692,107966213,136917841,230155,4078609217,539385431,1,20200930,164,26881097,653377382,30700,0,0,20315317,414778,1 +20201015 03:00,2,20180630,1996,6786715971,191291379350.360,110889933,149487211,363415,4292311678,539131901,1,20200930,203,38920543,943389448,34200,0,0,22216474,9450660,1 +20201016 03:00,1,20200930,230,43126619,1045477448,34200,0,0,25717236,9504159,1 +20201017 03:00,2,20191231,2274,6304381667,220028199194,113380430,143234619,185216,4316663027,556261501,1,20200930,266,48863061,1189252270,34200,11900,0,28807403,9524297,1 +20201020 03:00,1,20200930,317,61882346,1502914270,34200,11900,0,38577651,9557415,1 +20201021 03:00,2,20200630,2093,5921695548,141147549534.821,107384568,157707032,187805,4005128411,536801725,1,20200930,360,88438227,2142912270,34200,197330,0,64222447,9557415,1 +20201022 03:00,1,20200930,402,109122823,2637786270,136800,197330,0,83444326,9644813,1 +20201023 03:00,1,20200930,432,116005754,2803623907,141800,197330,0,89544309,9644813,1 +20201024 03:00,3,20181231,2056,6745154844,164760268747,231301128,226851472,195119,4605837602,479711918,1,20191231,2275,6304388667,220028446194,113380430,143234619,185216,4316663027,556261501,1,20200930,470,141627185,3420883402,141800,214351,0,109334322,9687522,1 +20201027 03:00,1,20200930,530,155588398,3758546927,1177000,397951,58812,122192023,9718057,1 +20201028 03:00,1,20200930,580,166927747,4034986927,1835400,851051,58812,130565585,9737617,1 +20201029 03:00,1,20200930,622,173534550,4194184927,1863900,892851,58812,133844364,12490340,1 +20201030 03:00,1,20200930,676,195312881,4718025927,1863900,892851,58812,148634565,12618239,1 +20201031 03:00,1,20200930,734,267350927,6422501015,1979165,1011640,58812,212317433,17955248,1 +20201103 03:00,2,20200630,2094,5921794705,141149904534.821,107384568,157832032,187805,4005227568,536801725,1,20200930,793,285109858,6850103015,1979165,1011640,198346,225553033,18416145,1 +20201104 03:00,1,20200930,866,300534503,7221669939,2025165,1017640,198346,233887842,19901723,1 +20201105 03:00,1,20200930,918,334528957,8039722179,2073665,1017640,198346,254998365,21960050,1 +20201106 03:00,1,20200930,986,417578801,10040517660,2366365,1859040,198346,326554050,21974910,1 +20201107 03:00,2,20200630,2095,5922353148,141163167534.821,107387568,157832032,187805,4005786010,536801725,1,20200930,1053,1006929903,23969974708,11376585,1859040,198346,825776842,30187580,1 +20201110 03:00,1,20200930,1149,1143385115,27244754208,12470485,1859040,198346,930613068,34150221,1 +20201111 03:00,2,20181231,2057,6745154844,164760268747,231301128,226851472,447859,4605837602,479711918,1,20200930,1237,1510749367,36094972525,12476425,1859618,368177,1254808865,39602911,1 +20201113 03:00,2,20191231,2276,6304394370,220028647194,113380430,143234619,185216,4316663356,556261501,1,20200930,1508,2255076839,53914098779,29795975,37626718,368177,1886887938,58611709,1 +20201114 03:00,2,20190930,2072,6327574142,184659369635,112544224,125444310,0,4338843468,435893534,1,20200930,1774,3289520295,78828283847,75736775,75098718,368233,2619818718,89143852,1 +20201117 03:00,2,20200630,2096,5922365593,141163463534.821,107387568,157832032,187805,4005786010,536801725,1,20200930,1994,5862651638,140806127242,121131275,153617618,1388986,4038714740,484979627,1 +20201118 03:00,1,20200930,2012,5882996830,141296265242,124841975,157712018,1388986,4058622407,485120593,1 +20201119 03:00,3,20200331,2060,5990881132,127423686692,107966213,136917841,230155,4078877990,539385431,1,20200630,2097,5923008515,141195674534.821,107387568,157832032,187805,4006428932,536801725,1,20200930,2016,5883093962,141298605242,124841975,157712018,1388986,4058683796,485120593,1 +20201120 03:00,1,20200930,2024,5883587298,141310490312,124842175,157712018,1388986,4058720890,485120593,1 +20201121 03:00,2,20200630,2098,5923039150,141196402125.821,107387568,157832032,187805,4006428932,536801725,1,20200930,2025,5883591082,141310580312,124842175,157712018,1388986,4058720890,485120593,1 +20201125 03:00,2,20200331,2061,5991520933,127440266692,107972013,137143641,230155,4079517791,539385431,1,20200930,2026,5883607072,141310965312,124842175,157712018,1388986,4058736580,485120593,1 +20201126 03:00,1,20200930,2028,5884109393,141323066312,124842175,157712018,1388986,4059209734,485120593,1 +20201201 03:00,3,20200331,2062,5991572877,127441369692,107972013,137143641,230155,4079517791,539385431,1,20200630,2099,5923090003,141197610125.821,107387568,157832032,187805,4006428932,536801725,1,20200930,2033,5885022783,141345077312,124842175,157712018,1388986,4059971644,485120593,1 +20201202 03:00,1,20200930,2034,5885045572,141346080312,124842175,157712018,1388986,4059993409,485120593,1 +20201203 03:00,1,20200930,2035,5887138735,141396504312,124842175,157712018,1388986,4062086572,485120593,1 +20201204 03:00,2,20191231,2277,6304417862,220029474194,113380430,143234619,185216,4316663356,556261501,1,20200930,2038,5888860464,141437980312,124842175,157712018,1388986,4063684097,485120593,1 +20201209 03:00,1,20200930,2039,5889159843,141445192312,124842175,157712018,1388986,4063983476,485120593,1 +20201211 03:00,1,20200930,2041,5889628903,141456492312,124842175,157712018,1388986,4064452536,485120593,1 +20201212 03:00,1,20200930,2043,5889779934,141460130312,124842175,157712018,1388986,4064603567,485120593,1 +20201215 03:00,1,20200930,2045,5894249451,141567801312,124842175,157712018,1388986,4069024287,485120593,1 +20201217 03:00,1,20200630,2100,5923234217,141201035125.821,107387568,157832032,187805,4006573146,536801725,1 +20201219 03:00,1,20200930,2047,5894279272,141568565312,124842175,157712018,1388986,4069036381,485120593,1 +20201222 03:00,1,20200930,2048,5894645823,141577395312,124842175,157712018,1388986,4069402932,485120593,1 +20201224 03:00,1,20200630,2101,5923254374,141201513125.821,107387568,157832032,187805,4006593303,536801725,1 +20201230 03:00,1,20200930,2049,5894666698,141577897312,124842175,157712018,1388986,4069423807,485120593,1 +20201231 03:00,4,20190331,2086,6598862462,182088190754,107128407,106386305,439507,4483483169,462573052,1,20190630,2065,6718314156,188067756453,86636645,104792254,316413,4616977048,461513468,1,20190930,2073,6327574142,184659369635,112544224,125444310,172371,4338843468,435893534,1,20200630,2102,5923254374,141201513125.821,107387568,157832032,358493,4006593303,536801725,1 diff --git a/output/alternative/sec/13f/bno.csv b/output/alternative/sec/13f/bno.csv new file mode 100644 index 0000000..70d6aab --- /dev/null +++ b/output/alternative/sec/13f/bno.csv @@ -0,0 +1,15 @@ +20201007 03:00,1,20200930,1,940,10000,0,0,0,940,0,0 +20201009 03:00,1,20200930,3,112419,1196888,0,0,0,112391,0,0 +20201015 03:00,1,20200930,4,141670,1507888,0,0,0,141642,0,0 +20201023 03:00,1,20200930,5,152720,1625888,0,0,0,152692,0,0 +20201028 03:00,1,20200930,6,152720,1625888,10100,3600,0,152692,0,0 +20201030 03:00,1,20200930,8,411573,4380888,10100,3600,0,162067,0,0 +20201103 03:00,1,20200930,9,490973,5226888,10100,3600,0,241467,0,0 +20201104 03:00,1,20200930,11,493897,5257888,10100,3600,0,241467,0,0 +20201105 03:00,1,20200930,13,3790183,39839888,10100,3600,0,275768,0,0 +20201106 03:00,1,20200930,15,3987561,41940888,10100,3600,0,441164,0,0 +20201110 03:00,1,20200930,16,4039083,42489888,10100,3600,0,441164,0,0 +20201113 03:00,1,20200930,24,4938986,52054888,10100,3600,0,1286839,7948,0 +20201114 03:00,1,20200930,32,5090083,53659888,147100,87900,0,1348813,7948,0 +20201117 03:00,1,20200930,37,7893886,83519888,230500,232700,0,4152516,8048,0 +20201118 03:00,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 diff --git a/output/alternative/sec/13f/eem.csv b/output/alternative/sec/13f/eem.csv new file mode 100644 index 0000000..f67d06c --- /dev/null +++ b/output/alternative/sec/13f/eem.csv @@ -0,0 +1,50 @@ +20201002 03:00,1,20200930,4,75204,3315000,0,0,0,67226,0,0 +20201003 03:00,1,20200930,5,242449,10689000,0,0,0,217775,0,1 +20201006 03:00,1,20200930,13,3308578,145875000,0,0,0,3192434,0,1 +20201007 03:00,1,20200930,25,3497442,154200000,0,0,0,3323223,0,1 +20201008 03:00,3,20200331,1137,448549380,15307742974,197189119,327837638,1096248,339359915,29758743,1,20200630,1094,402227786,16094457604,162866492,266937301,941231,298688854,30555320,1,20200930,34,3703189,163258000,0,0,0,3360744,2808,1 +20201009 03:00,1,20200930,43,4124808,181846798,0,0,0,3763761,2808,1 +20201010 03:00,1,20200930,55,7271037,320562798,0,100,0,6620726,2808,1 +20201014 03:00,3,20171231,1135,552028727,25290479713,136878265,272461703,770961,432279700,51624463,1,20200630,1095,402262479,16095967604,162866492,266937301,941231,298688854,30555320,1,20200930,82,8584156,378460798,0,100,0,7522690,17199,1 +20201015 03:00,1,20200930,104,10102847,445425798,5000,100,0,8714631,17199,1 +20201016 03:00,1,20200930,119,12409155,547301798,5000,100,0,10532580,39782,1 +20201017 03:00,1,20200930,138,13066953,576414798,78900,144900,0,10649854,39782,1 +20201020 03:00,1,20200930,160,13505916,595772798,78900,144900,0,10962497,39782,1 +20201021 03:00,1,20200930,184,30776151,1357213798,78900,144900,0,27817219,39782,1 +20201022 03:00,1,20200930,202,31294089,1377641883,78900,144900,0,28188813,40207,1 +20201023 03:00,1,20200930,220,32815323,1444712883,78900,144900,0,29202316,40207,1 +20201024 03:00,1,20200930,246,33444576,1472473883,878900,144900,0,29588429,40207,1 +20201027 03:00,1,20200930,279,34313850,1509227292,2195900,161600,8063,29972928,53957,1 +20201028 03:00,1,20200930,305,35458793,1559657292,3144700,2044000,8063,30582085,53957,1 +20201029 03:00,1,20200930,326,36827580,1620005292,3144700,2044000,8063,31497296,56746,1 +20201030 03:00,1,20200930,355,44167941,1943657292,3144700,2044000,8063,37589953,112180,1 +20201031 03:00,1,20200930,395,52503296,2312065481,3163944,2078809,388547,42598084,2655622,1 +20201103 03:00,1,20200930,414,53656506,2362870481,3164344,2108709,388547,43227369,2655622,1 +20201104 03:00,1,20200930,458,57503743,2532495348,3172344,2116709,395147,43864367,2698842,1 +20201105 03:00,1,20200930,488,60587466,2668470464,3246644,2123709,395147,44986379,2710198,1 +20201106 03:00,2,20200630,1096,402276414,16096524604,162866492,266937301,941231,298688854,30555320,1,20200930,525,79579772,3505837834,8802244,8538905,395147,59990446,2716815,1 +20201107 03:00,1,20200930,563,159398006,7026027341,10982744,8538920,395147,138221250,3383968,1 +20201110 03:00,1,20200930,611,174296779,7682964827,10984544,8541720,395147,146185308,3421847,1 +20201111 03:00,2,20181231,1210,513478451,20214091478,198367094,269321579,749384,393243915,31843233,1,20200930,676,189056625,8333895191,10985396,8660843,395147,159274840,3654000,1 +20201113 03:00,1,20200930,824,266533266,11736518249,86036396,65758343,395147,214685104,14565259,1 +20201114 03:00,2,20200630,1097,402308705,16097815604,162866492,266937301,941231,298688854,30555320,1,20200930,983,377110091,16597398369,134371996,220280743,961076,278668597,27473637,1 +20201117 03:00,1,20200930,1100,417251323,18366585369,165745296,273437743,961076,311124798,32950852,1 +20201118 03:00,1,20200930,1110,418906161,18439585369,175912396,282681143,961076,312649177,32950852,1 +20201119 03:00,4,20191231,1324,501665868,22495996706,200441151,292309788,422047,373699660,35616650,1,20200331,1138,448565031,15308276974,197189119,327837638,1096248,339359915,29758743,1,20200630,1098,402324068,16098429604,162866492,266937301,941231,298688854,30555320,1,20200930,1116,419273124,18455765369,175912396,282685743,961076,312888545,32950852,1 +20201120 03:00,1,20200930,1119,419361170,18459646971,175912396,282685743,961076,312888545,32950852,1 +20201121 03:00,2,20200331,1139,448646486,15311057041,197189119,327837638,1096248,339359915,29758743,1,20200630,1099,402400378,16101481244,162866492,266937301,941231,298688854,30555320,1 +20201124 03:00,1,20200930,1121,419395060,18461140971,175912396,282685743,961076,312888545,32950877,1 +20201125 03:00,1,20200930,1122,419402240,18461457971,175912396,282685743,961076,312895725,32950877,1 +20201126 03:00,1,20200930,1125,419827043,18480186971,175912396,282685743,961076,313169437,32950877,1 +20201128 03:00,1,20200930,1126,419889067,18482921971,175912396,282685743,961076,313169437,32950877,1 +20201201 03:00,1,20200930,1128,419915680,18484105971,175912396,282685743,961076,313171187,32950877,1 +20201202 03:00,2,20200630,1100,402425778,16102500244,162866492,266937301,941231,298688854,30555320,1,20200930,1129,419920918,18484336971,175912396,282685743,961076,313174990,32950877,1 +20201204 03:00,1,20200930,1131,419967138,18486374971,175912396,282685743,961076,313221172,32950877,1 +20201211 03:00,1,20200930,1133,419994183,18487566971,175912396,282685743,961076,313248217,32950877,1 +20201212 03:00,1,20200930,1135,420027588,18489040971,175912396,282685743,961076,313281583,32950877,1 +20201215 03:00,1,20200930,1137,420825306,18524211971,175912396,282685743,961076,314061163,32950877,1 +20201217 03:00,1,20200630,1101,402437245,16102959244,162866492,266937301,941231,298700321,30555320,1 +20201219 03:00,1,20200331,1140,448738858,15314210041,197189119,327837638,1096248,339452287,29758743,1 +20201222 03:00,1,20200930,1138,420876787,18526481971,175912396,282685743,961076,314112644,32950877,1 +20201224 03:00,1,20200630,1102,402529494,16106648244,162866492,266937301,941231,298792570,30555320,1 +20201230 03:00,1,20200930,1139,420970464,18530611971,175912396,282685743,961076,314206321,32950877,1 diff --git a/output/alternative/sec/13f/goog.csv b/output/alternative/sec/13f/goog.csv new file mode 100644 index 0000000..a82e826 --- /dev/null +++ b/output/alternative/sec/13f/goog.csv @@ -0,0 +1,54 @@ +20201002 03:00,1,20200930,8,39809,58503000,0,0,0,18575,19,0 +20201003 03:00,2,20161231,1899,243087213,187575011592,5177601,5493214,9380,155507842,14456041,1,20200930,15,66166,97237000,0,0,0,31431,19,1 +20201006 03:00,1,20200930,25,163841,240678000,0,0,0,129004,19,1 +20201007 03:00,3,20170331,1938,242942183,201249420540,6155364,4834277,220,151491003,14254829,1,20170930,1974,243248119,233337108012,10724085,6747095,321,148817834,19108118,1,20200930,46,248706,365044000,0,0,0,182194,39,1 +20201008 03:00,4,20170630,1984,245226494,219983428632,8643962,6235978,220,163268621,14555598,1,20200331,2542,225103114,260425437125,2996132,4151956,973,148882205,12705514,1,20200630,2578,224501641,311708424305,3153704,4007761,864,144998537,12179536,1,20200930,57,491270,721519000,0,0,0,420408,139,1 +20201009 03:00,1,20200930,87,655988,963589080,0,0,0,532405,3020,1 +20201010 03:00,1,20200930,119,923050,1356033080,0,0,0,725848,3853,1 +20201014 03:00,5,20171231,2160,242471223,253740503082,11817742,8289959,169,164489773,18511344,1,20180331,2144,260047432,247640637006,7560608,7102438,169,173678381,36763824,1,20180930,2226,284350133,310433457410,4595710,5069892,125,172444743,19344922,1,20200331,2543,225103114,260425437125,2996132,4151956,1018,148882205,12705514,1,20200930,194,1146839,1685005080,1600,0,0,845543,12861,1 +20201015 03:00,2,20180630,2182,241784765,267589802698,5069369,5074423,142,165115242,18150734,1,20200930,237,1396842,2052388826,1600,0,0,1005680,62587,1 +20201016 03:00,1,20200930,276,1645069,2417205412,1600,0,0,1157940,149291,1 +20201017 03:00,2,20191231,2586,230446345,307824272711,4413939,4735835,125,153516061,13840046,1,20200930,320,1813996,2665347412,1600,400,0,1264449,149511,1 +20201020 03:00,1,20200930,390,2461129,3616362412,1610,400,0,1818993,150924,1 +20201021 03:00,2,20200630,2579,224586503,311828423305,3153704,4007761,864,145083399,12179536,1,20200930,443,3160050,4642606412,1610,7556,0,2449520,150924,1 +20201022 03:00,1,20200930,489,3592913,5278700412,1610,7556,0,2846527,151111,1 +20201023 03:00,1,20200930,531,3857137,5667178412,1610,7556,0,3020398,151111,1 +20201024 03:00,2,20181231,2342,363669609,248397476740,5646101,6171250,0,225539640,17685069,1,20200930,585,4113133,6043500092,1610,8436,0,3229291,154184,1 +20201027 03:00,1,20200930,657,4584703,6737058274,1610,9836,1378,3660754,171511,1 +20201028 03:00,1,20200930,718,5237393,7690343274,5510,11736,1378,4214064,172162,1 +20201029 03:00,1,20200930,771,5638175,8279094274,9110,11736,1378,4402980,351518,1 +20201030 03:00,1,20200930,839,6312860,9269540274,9110,12336,1378,4905197,359282,1 +20201031 03:00,1,20200930,910,10010129,14713986654,10331,13343,1378,8190475,632294,1 +20201103 03:00,1,20200930,981,11508876,16912314953,10331,13343,1378,9180479,648243,1 +20201104 03:00,1,20200930,1072,12375745,18186244328,10331,13343,1378,9744835,726095,1 +20201105 03:00,1,20200930,1134,14821286,21790313774,10331,13343,1378,11716664,819985,1 +20201106 03:00,2,20200630,2580,224586777,311828810305,3153704,4007761,864,145083673,12179536,1,20200930,1221,19001593,27933888098,10331,13543,1378,15347468,833230,1 +20201107 03:00,2,20200630,2581,224587604,311829979305,3153704,4007761,864,145084500,12179536,1,20200930,1302,42712028,62161229912,10336,13543,1378,35201649,1193874,1 +20201110 03:00,1,20200930,1426,47695125,69490051903,10336,13543,1378,39104506,1213678,1 +20201111 03:00,2,20181231,2343,363669609,248397476740,5646101,6171250,169,225539640,17685069,1,20200930,1555,62436874,91154786589,10336,13543,1434,51527443,1555747,1 +20201113 03:00,6,20190331,2381,239011324,280103059844,3400907,3564193,0,180076277,18071775,1,20190630,2343,237327246,255514478866,4002109,4691051,2796,153689246,17217479,1,20190930,2382,233159388,284195335689,4592378,4408517,1773,162993883,16798870,1,20191231,2587,230446566,307824567711,4413939,4735835,125,153516073,13840046,1,20200331,2544,225103370,260425735125,2996132,4151956,1018,148882228,12705514,1,20200930,1903,99422358,145587926836,1497984,603289,1434,80305160,2904420,1 +20201114 03:00,2,20190930,2383,234977179,286411222689,4592378,4408517,1773,164811674,16798870,1,20200930,2260,147782941,216526221953,3824884,2117989,1434,114669405,4503653,1 +20201117 03:00,1,20200930,2538,215033575,315369159923,9144704,5588289,1434,140109348,12185080,1 +20201118 03:00,1,20200930,2559,215750239,316422676923,9583404,5812989,1434,140798562,12191671,1 +20201119 03:00,4,20191231,2588,230446592,307824602711,4413939,4735835,125,153516073,13840046,1,20200331,2545,225113359,260456742125,2996132,4151956,1018,148892217,12705514,1,20200630,2582,224603906,311878593305,3153704,4007761,864,145100802,12179536,1,20200930,2565,215765803,316445550923,9583404,5812989,1434,140810632,12191671,1 +20201120 03:00,1,20200930,2574,215776426,316461147561,9583404,5812989,1434,140811875,12191671,1 +20201124 03:00,3,20200331,2546,225113542,260456955125,2996132,4151956,1018,148892217,12705514,1,20200630,2583,224604067,311878821305,3153704,4007761,864,145100802,12179536,1,20200930,2575,215776732,316461596561,9583404,5812989,1434,140811875,12191671,1 +20201125 03:00,1,20200930,2576,215777254,316462363561,9583404,5812989,1434,140812382,12191671,1 +20201126 03:00,1,20200930,2578,215790908,316482429561,9583404,5812989,1434,140825792,12191671,1 +20201128 03:00,1,20200930,2580,215798936,316494114561,9583404,5812989,1434,140825792,12191671,1 +20201201 03:00,3,20200331,2547,225115460,260459185125,2996132,4151956,1018,148892217,12705514,1,20200630,2584,224606107,311881705305,3153704,4007761,864,145100802,12179536,1,20200930,2583,215801107,316497337561,9583404,5812989,1434,140826116,12191671,1 +20201202 03:00,1,20200930,2585,215813063,316514896561,9583404,5812989,1434,140837806,12191671,1 +20201203 03:00,1,20200930,2586,215870995,316600033561,9583404,5812989,1434,140895738,12191671,1 +20201204 03:00,2,20191231,2589,230448035,307826531711,4413939,4735835,125,153516073,13840046,1,20200930,2590,215899437,316641831561,9583404,5812989,1434,140913262,12191671,1 +20201208 03:00,1,20200930,2591,215900213,316643067561,9583404,5812989,1434,140913293,12191671,1 +20201209 03:00,1,20200930,2592,215907806,316654226561,9583404,5812989,1434,140920886,12191671,1 +20201211 03:00,1,20200930,2594,215916894,316667581561,9583404,5812989,1434,140929974,12191671,1 +20201212 03:00,1,20200930,2597,215928637,316684839561,9583404,5812989,1434,140939167,12191671,1 +20201215 03:00,1,20200930,2599,216126116,316975054561,9583404,5812989,1434,141135478,12191671,1 +20201216 03:00,2,20200630,2585,224608917,311885677305,3153704,4007761,864,145103612,12179536,1,20200930,2600,216128833,316979047561,9583404,5812989,1434,141138195,12191671,1 +20201217 03:00,1,20200630,2586,224617399,311897667305,3153704,4007761,864,145112094,12179536,1 +20201219 03:00,1,20200930,2601,216136920,316993197561,9583404,5812989,1434,141138195,12191671,1 +20201222 03:00,1,20200930,2602,216185603,317064742561,9583404,5812989,1434,141186726,12191671,1 +20201224 03:00,1,20200630,2587,224617578,311897920305,3153704,4007761,864,145112273,12179536,1 +20201230 03:00,1,20200930,2603,216185801,317065032561,9583404,5812989,1434,141186924,12191671,1 +20201231 03:00,4,20190331,2382,239011324,280103059844,3400907,3564193,125,180076277,18071775,1,20190630,2344,237327246,255514478866,4002109,4691051,2921,153689246,17217479,1,20190930,2384,234977179,286411222689,4592378,4408517,1898,164811674,16798870,1,20200630,2588,224617578,311897920305,3153704,4007761,911,145112273,12179536,1 diff --git a/output/alternative/sec/13f/googl.csv b/output/alternative/sec/13f/googl.csv new file mode 100644 index 0000000..ec883d4 --- /dev/null +++ b/output/alternative/sec/13f/googl.csv @@ -0,0 +1,55 @@ +20201002 03:00,1,20200930,4,55446,81261000,0,0,0,41379,0,0 +20201003 03:00,2,20161231,2023,240806833,190576489611,4711850,5182867,2452,154449768,19422639,1,20200930,12,72383,106086000,0,0,0,54170,0,1 +20201006 03:00,1,20200930,23,148893,218219000,0,0,0,130595,0,1 +20201007 03:00,3,20170331,2064,241507881,203527186639,3570888,4982430,333,153568095,18505019,1,20170930,2079,236868551,230555573124,6204052,6349800,855,159289883,20635641,1,20200930,49,232394,340531080,0,0,0,202023,61,1 +20201008 03:00,4,20170630,2090,239793626,221157981257,5355982,6113267,2082,161536678,19292863,1,20200331,2770,233854352,268867659421,5750476,6775270,6856,153847404,18463242,1,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,64,614898,901129080,0,0,0,573539,73,1 +20201009 03:00,1,20200930,99,861799,1263114502,0,0,0,777642,1347,1 +20201010 03:00,1,20200930,137,1184258,1735711502,0,0,0,1038573,1580,1 +20201014 03:00,5,20171231,2303,239141993,251756678574,5880758,6860420,775,157511943,20289648,1,20180331,2305,253623721,243920570704,6604287,7925357,12422,163434073,38089952,1,20180930,2373,237927432,283353979377,5421067,5878238,2229,149283789,21799754,1,20200331,2771,233854352,268867659421,5750476,6775270,7056,153847404,18463242,1,20200930,221,1718755,2519254502,0,0,0,1545929,12625,1 +20201015 03:00,2,20180630,2345,238450757,265891172973,6483657,6360726,1859,158966468,21371904,1,20200930,272,2260723,3307265083,0,0,0,1685849,362483,1 +20201016 03:00,1,20200930,311,2751843,4027110083,0,0,0,2006533,489612,1 +20201017 03:00,2,20191231,2807,241185106,318861919329,6198466,7621857,547,160970497,19815509,1,20200930,359,2925622,4280764083,0,1800,0,2140774,490989,1 +20201020 03:00,1,20200930,431,4467195,6539167083,10500,1800,0,3602077,491501,1 +20201021 03:00,1,20200930,492,5329928,7803748083,10900,7675,0,4402043,491501,1 +20201022 03:00,1,20200930,545,6126127,8970702083,10900,7675,0,5116428,492161,1 +20201023 03:00,1,20200930,589,6384926,9350745012,10900,7675,0,5324010,492161,1 +20201024 03:00,2,20181231,2512,359354819,313078781035,5340971,6259415,0,269305176,19419436,1,20200930,646,6650093,9740280816,10900,8934,0,5542366,498329,1 +20201027 03:00,1,20200930,722,7356397,10774933938,32162,13496,418,6187982,500381,1 +20201028 03:00,1,20200930,793,7630943,11171654938,35862,22996,418,6407786,500504,1 +20201029 03:00,1,20200930,852,8033133,11761241938,40362,46196,418,6657749,612388,1 +20201030 03:00,1,20200930,920,9419513,13791223335,40362,46196,418,7681486,621897,1 +20201031 03:00,1,20200930,995,12620305,18492827100,40788,47915,418,10351214,912339,1 +20201103 03:00,1,20200930,1076,13488765,19761846541,40788,47915,4026,10950984,916433,1 +20201104 03:00,1,20200930,1179,14993874,21967601002,40788,47915,4026,11959590,1140973,1 +20201105 03:00,1,20200930,1253,17984427,26351193487,40788,47915,4026,14335219,1252829,1 +20201106 03:00,1,20200930,1344,23668795,34681915324,59588,48915,4026,18958287,1262897,1 +20201107 03:00,2,20200630,2811,233475674,327058162860,5357763,5656871,5140,152227084,18129403,1,20200930,1432,48183674,69998358829,109689,48915,4026,39364575,1639975,1 +20201110 03:00,1,20200930,1553,54442424,79155715322,109689,59215,4026,44189362,1802598,1 +20201111 03:00,2,20181231,2513,359354819,313078781035,5340971,6259415,210,269305176,19419436,1,20200930,1692,70696311,102979660549,112289,59218,4237,63348499,2102442,1 +20201113 03:00,1,20200930,2068,112086118,155107805246,1087289,846218,11053,97633690,3404534,1 +20201114 03:00,3,20190930,2540,236538496,288491815615,5430800,6644989,0,170120016,20175294,1,20191231,2808,241185505,318862453329,6198466,7621857,547,160970497,19815509,1,20200930,2443,171109889,238726546549,2292889,2272918,11926,135958289,5157054,1 +20201117 03:00,3,20200331,2772,233854705,268868069421,5750476,6775270,7056,153847404,18463242,1,20200630,2813,233476899,327059899860,5357763,5656871,5140,152227084,18129403,1,20200930,2758,235083685,332102492865,7436089,7149018,11926,156818072,18774703,1 +20201118 03:00,1,20200930,2781,236061761,333536431865,7831689,7593118,11926,157739890,18783016,1 +20201119 03:00,4,20191231,2809,241185531,318862488329,6198466,7621857,547,160970497,19815509,1,20200331,2773,233857772,268877582421,5750476,6775270,7056,153850471,18463242,1,20200630,2814,233479966,327069074860,5357763,5656871,5140,152230151,18129403,1,20200930,2789,236075135,333556031865,7831689,7593118,11926,157750452,18783016,1 +20201120 03:00,1,20200930,2799,236098098,333589686815,7831689,7609218,11926,157752655,18783016,1 +20201121 03:00,1,20200930,2800,236100494,333593558815,7831689,7609218,11926,157752655,18783016,1 +20201124 03:00,1,20200930,2801,236102765,333596886815,7831689,7609218,11926,157752655,18783016,1 +20201125 03:00,1,20200930,2802,236104360,333599224815,7831689,7609218,11926,157754201,18783016,1 +20201126 03:00,1,20200930,2804,236122427,333625703815,7831689,7609218,11926,157771985,18783016,1 +20201128 03:00,1,20200930,2805,236122832,333626297815,7831689,7609218,11926,157771985,18783016,1 +20201201 03:00,3,20200331,2774,233858199,268878078421,5750476,6775270,7056,153850471,18463242,1,20200630,2815,233480386,327069670860,5357763,5656871,5140,152230151,18129403,1,20200930,2808,236124187,333628351815,7831689,7609218,11926,157772196,18783016,1 +20201202 03:00,1,20200930,2810,236172416,333699044815,7831689,7609218,11926,157819357,18783016,1 +20201203 03:00,1,20200930,2811,236261902,333830195815,7831689,7609218,11926,157908843,18783016,1 +20201204 03:00,2,20191231,2810,241185898,318862980329,6198466,7621857,547,160970497,19815509,1,20200930,2815,236279727,333856318815,7831689,7609218,11926,157921919,18783016,1 +20201208 03:00,1,20200930,2816,236280041,333856816815,7831689,7609218,11926,157921942,18783016,1 +20201209 03:00,1,20200930,2817,236283166,333861396815,7831689,7609218,11926,157925067,18783016,1 +20201211 03:00,1,20200930,2819,236294670,333878247815,7831689,7609218,11926,157936571,18783016,1 +20201212 03:00,1,20200930,2823,236329682,333929561815,7831689,7609218,11926,157970286,18783016,1 +20201215 03:00,1,20200930,2826,236540485,334238514815,7831689,7609218,11926,158162819,18783016,1 +20201216 03:00,2,20200630,2816,233482884,327073212860,5357763,5656871,5140,152232649,18129403,1,20200930,2827,236542910,334242068815,7831689,7609218,11926,158165244,18783016,1 +20201217 03:00,1,20200630,2817,233489492,327082582860,5357763,5656871,5140,152239257,18129403,1 +20201219 03:00,2,20200331,2775,233858884,268878874421,5750476,6775270,7056,153851156,18463242,1,20200930,2829,236548989,334251405815,7831689,7609218,11926,158169778,18783016,1 +20201222 03:00,1,20200930,2830,236576911,334292327815,7831689,7609218,11926,158193673,18783016,1 +20201224 03:00,1,20200630,2818,233489891,327083147860,5357763,5656871,5140,152239656,18129403,1 +20201230 03:00,1,20200930,2831,236577253,334292828815,7831689,7609218,11926,158194015,18783016,1 +20201231 03:00,4,20190331,2561,236698511,278170111068,4092046,5151292,460,182746951,19639545,1,20190630,2510,241794170,256771384398,6200189,6928081,200,154654106,20383850,1,20190930,2541,236538496,288491815615,5430800,6644989,200,170120016,20175294,1,20200630,2819,233489891,327083147860,5357763,5656871,5342,152239656,18129403,1 diff --git a/output/alternative/sec/13f/ibm.csv b/output/alternative/sec/13f/ibm.csv new file mode 100644 index 0000000..018d2f1 --- /dev/null +++ b/output/alternative/sec/13f/ibm.csv @@ -0,0 +1,50 @@ +20201002 03:00,1,20200930,8,148094,18019000,0,0,0,83301,0,0 +20201003 03:00,2,20161231,1831,565053411,93712654698,11895307,12945374,2829,380179160,31602351,1,20200930,12,177831,21638000,0,0,0,104638,0,1 +20201006 03:00,1,20200930,23,434335,53051000,0,0,0,299645,1052,1 +20201007 03:00,2,20170331,1865,555598316,96516263584,11206235,9948012,350,366035591,31273081,1,20200930,45,752019,91733544,0,0,0,454324,1186,1 +20201008 03:00,4,20170630,1791,550381395,84431903792,9527043,11240110,350,351636775,32930754,1,20200331,1937,503902209,55341397580,12204813,13803017,4794,302445317,43325985,1,20200630,1947,506089972,60863567152,10227584,12215460,32374,302621172,43346758,1,20200930,57,823509,100425544,0,0,0,483097,3957,1 +20201009 03:00,1,20200930,84,1304767,158980544,0,0,0,788482,8973,1 +20201010 03:00,1,20200930,116,2157748,262781544,1,0,0,1322040,9801,1 +20201014 03:00,6,20171231,1964,527560371,80915477613,11179462,13869557,1592,326345224,34559402,1,20180331,1958,525561005,80335395427,12118339,13426595,45075,365665845,35294947,1,20180930,1919,525422158,79062542207,13138025,13535534,221674,343335018,37897073,1,20200331,1938,503902209,55341397580,12204813,13803017,30297,302445317,43325985,1,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,178,3091325,376468544,1,0,0,1947058,11467,1 +20201015 03:00,2,20180630,1940,515512117,71883743548,11090006,14611146,35880,318732417,35470432,1,20200930,214,3667122,446506052,1,0,0,2222289,136014,1 +20201016 03:00,1,20200930,241,4131114,502976052,1,0,0,2591105,136164,1 +20201017 03:00,2,20191231,2045,506797062,67813336105,8897585,12736422,25889,312562480,46722628,1,20200930,270,4707297,573067052,1,0,0,2991007,139851,1 +20201020 03:00,1,20200930,314,6092234,741566052,73,197,0,3816802,149285,1 +20201021 03:00,1,20200930,356,8553412,1041040052,73,22059,0,6037148,149285,1 +20201022 03:00,1,20200930,398,10055303,1214978052,4273,22059,0,7284722,161373,1 +20201023 03:00,1,20200930,430,10481806,1267036052,4273,22059,0,7485854,161373,1 +20201024 03:00,2,20181231,1899,549439644,56405288451,11957201,12529781,2595,368976070,36185264,1,20200930,469,14175100,1716306623,4273,26339,0,9658494,172175,1 +20201027 03:00,1,20200930,523,15662443,1897313306,107173,105539,14141,10963081,177897,1 +20201028 03:00,1,20200930,572,16881356,2047206306,166773,112939,14141,11913440,179797,1 +20201029 03:00,1,20200930,606,18037575,2187906306,166773,112939,14141,12481487,474656,1 +20201030 03:00,1,20200930,660,22725288,2757410306,166773,112939,14141,15397644,507407,1 +20201031 03:00,1,20200930,720,31889412,3870733925,169725,122219,14141,23165399,1391740,1 +20201103 03:00,1,20200930,778,33672083,4086225926,170225,122232,14141,24518596,1399455,1 +20201104 03:00,1,20200930,852,35273301,4271845971,170225,122232,14141,25649825,1407630,1 +20201105 03:00,1,20200930,906,42599141,5162938156,187325,122232,14141,31016865,1544414,1 +20201106 03:00,1,20200930,971,58332004,7077118156,205625,190832,14141,45149476,1564231,1 +20201107 03:00,2,20200630,1949,506133620,60869098152,10227584,12215460,32374,302631960,43346758,1,20200930,1043,129771803,15619935965,206673,190876,14141,103730857,3248883,1 +20201110 03:00,1,20200930,1125,144724228,17439563354,206673,190876,14141,112316531,3624605,1 +20201111 03:00,2,20181231,1900,549439644,56405288451,11957201,12529781,4187,368976070,36185264,1,20200930,1214,201961196,24403463612,206673,200876,38428,151692949,12139456,1 +20201113 03:00,1,20200930,1476,267828177,32393190765,902373,1544276,38428,203713089,14260527,1 +20201114 03:00,4,20190930,1946,504593578,73288798787,11425479,12574385,1167,320353602,44223676,1,20191231,2046,506799679,67813687105,8897585,12736422,25889,312562480,46722628,1,20200630,1950,506145466,60870490152,10227584,12215460,32374,302631960,43346758,1,20200930,1723,368808059,44667481135,3826473,6089976,38557,275925073,17966388,1 +20201117 03:00,3,20200331,1939,503905879,55341804580,12204813,13803017,30297,302445317,43325985,1,20200630,1951,506149136,60870933152,10227584,12215460,32374,302631960,43346758,1,20200930,1894,494513673,59960517738,9657073,12334776,46773,299507638,40577520,1 +20201118 03:00,1,20200930,1912,496866517,60246711738,10596073,12966776,46773,301619567,40758566,1 +20201119 03:00,4,20191231,2047,506799709,67813691105,8897585,12736422,25889,312562480,46722628,1,20200331,1940,503905909,55341807580,12204813,13803017,30297,302445317,43325985,1,20200630,1953,506157022,60872938152,10227584,12215460,32374,302639816,43346758,1,20200930,1919,496908029,60251763738,10596073,12966776,46773,301649518,40758566,1 +20201120 03:00,1,20200930,1925,496944801,60256237278,10596173,12966876,46773,301653878,40758566,1 +20201121 03:00,1,20200930,1926,496944829,60256240278,10596173,12966876,46773,301653878,40758566,1 +20201124 03:00,3,20200331,1941,503907877,55342025580,12204813,13803017,30297,302445317,43325985,1,20200630,1954,506158870,60873161152,10227584,12215460,32374,302639816,43346758,1,20200930,1927,496947686,60256586278,10596173,12966876,46773,301653878,40758566,1 +20201125 03:00,1,20200930,1928,496951885,60257097278,10596173,12966876,46773,301658077,40758566,1 +20201126 03:00,1,20200930,1929,496959656,60258042278,10596173,12966876,46773,301658077,40758566,1 +20201128 03:00,1,20200930,1930,496964184,60258593278,10596173,12966876,46773,301658077,40758566,1 +20201201 03:00,3,20200331,1942,503912092,55342493580,12204813,13803017,30297,302445317,43325985,1,20200630,1955,506163088,60873670152,10227584,12215460,32374,302639816,43346758,1,20200930,1933,496988716,60261622278,10596173,12966876,46773,301674708,40758566,1 +20201202 03:00,1,20200930,1934,496997327,60262670278,10596173,12966876,46773,301683219,40758566,1 +20201203 03:00,1,20200930,1935,497076908,60272353278,10596173,12966876,46773,301762800,40758566,1 +20201204 03:00,2,20191231,2048,506802653,67814086105,8897585,12736422,25889,312562480,46722628,1,20200930,1939,497153527,60281674278,10596173,12966876,46773,301785500,40758566,1 +20201209 03:00,1,20200930,1940,497161383,60282630278,10596173,12966876,46773,301793356,40758566,1 +20201211 03:00,1,20200930,1942,497221340,60289925278,10596173,12966876,46773,301853313,40758566,1 +20201212 03:00,1,20200930,1945,497270879,60295952278,10596173,12966876,46773,301871002,40758566,1 +20201215 03:00,1,20200930,1948,497735599,60352494278,10596173,12966876,46773,302287242,40758566,1 +20201217 03:00,1,20200630,1956,506177667,60875431152,10227584,12215460,32374,302654395,43346758,1 +20201222 03:00,1,20200930,1949,497895752,60371980278,10596173,12966876,46773,302447358,40758566,1 +20201231 03:00,4,20190331,1966,500979098,70469005324,12436348,15814940,41798,324887639,39866644,1,20190630,1932,501013636,70706282237,11027053,13621970,28390,311317759,42941503,1,20190930,1947,504593578,73288798787,11425479,12574385,24750,320353602,44223676,1,20200630,1957,506177667,60875431152,10227584,12215460,56886,302654395,43346758,1 diff --git a/output/alternative/sec/13f/iwm.csv b/output/alternative/sec/13f/iwm.csv new file mode 100644 index 0000000..daedfa3 --- /dev/null +++ b/output/alternative/sec/13f/iwm.csv @@ -0,0 +1,49 @@ +20201002 03:00,1,20200930,3,10158,1521000,0,0,0,3900,0,0 +20201003 03:00,1,20200930,6,264998,39694000,0,0,0,253677,0,1 +20201006 03:00,1,20200930,13,401407,60127000,0,0,0,365776,0,1 +20201007 03:00,1,20200930,31,701402,105086712,0,0,0,395122,430,1 +20201008 03:00,2,20200331,1464,294819699,33700126410,105016517,276471475,84822,209150925,31507932,1,20200930,42,809407,121291712,0,0,0,447795,2238,1 +20201009 03:00,1,20200930,61,1252939,187729824,0,0,0,868908,2238,1 +20201010 03:00,1,20200930,75,4494500,673280824,0,0,0,3897875,2238,1 +20201014 03:00,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,119,7224789,1082332824,0,0,0,5698818,169054,1 +20201015 03:00,1,20200930,149,9197080,1377715000,0,0,0,7401226,169054,1 +20201016 03:00,1,20200930,171,9904049,1483616000,0,0,0,7935522,172763,1 +20201017 03:00,1,20200930,199,11426506,1711960000,45600,24000,0,8398019,176674,1 +20201020 03:00,1,20200930,230,11766925,1762950000,47024,25094,0,8591641,176975,1 +20201021 03:00,1,20200930,261,12881970,1929974000,47024,25094,0,8844839,176975,1 +20201022 03:00,1,20200930,297,14603766,2029839000,47024,25094,0,10304633,176975,1 +20201023 03:00,1,20200930,322,16144149,2260812432,47024,25094,0,11379444,176975,1 +20201024 03:00,1,20200930,351,16835333,2364361432,47024,35048,0,11790739,265014,1 +20201027 03:00,1,20200930,397,17620529,2482042626,523324,144848,19850,12104866,265230,1 +20201028 03:00,1,20200930,429,18463120,2608237626,1225624,996748,19850,12605749,265232,1 +20201029 03:00,1,20200930,456,19078232,2700376626,1225624,1007348,19850,12875834,265652,1 +20201030 03:00,1,20200930,496,27328552,3919458626,1225624,1007348,19850,20636875,284463,1 +20201031 03:00,1,20200930,542,37124553,5389247894,1226632,1028514,90100,22039115,7997034,1 +20201103 03:00,1,20200930,579,38025744,5511358907,1226732,1580814,90100,22367693,8000335,1 +20201104 03:00,1,20200930,638,39615389,5749440908,2076032,1819214,90100,22989258,8019610,1 +20201105 03:00,1,20200930,675,42176620,6133086908,2088832,1839214,90100,24170364,8098044,1 +20201106 03:00,1,20200930,730,70268332,10340973652,5697832,11870318,90100,50031589,9272715,1 +20201107 03:00,2,20200630,1521,296188765,42214674601,163309923,274619253,258484,200044339,23736139,1,20200930,781,93602526,13836541492,5897840,13293721,90100,71265796,10241510,1 +20201110 03:00,1,20200930,839,101739943,15056316861,5923690,13300321,90100,74885241,10274675,1 +20201111 03:00,1,20200930,916,107194763,15873277195,5939690,13383377,90100,88241958,10330204,1 +20201113 03:00,4,20181231,1431,261691027,35049637886,78679900,139483089,135961,208943538,17581144,1,20190331,1427,259738726,39672565470,66830673,179029780,107879,183922678,26997274,1,20190630,1426,251649073,38714998959,80328510,145570244,124826,181523458,21009177,1,20200930,1097,162300377,24123452741,31753390,92463033,92269,130510320,16755092,1 +20201114 03:00,2,20200630,1522,296194135,42215443601,163309923,274619253,258484,200044339,23736139,1,20200930,1295,264730189,39438609932,71609190,184060004,92269,165275688,28534807,1 +20201117 03:00,1,20200930,1473,285448404,42543352932,134723990,299100279,155421,183312946,29508001,1 +20201118 03:00,1,20200930,1489,286059255,42635199932,142405490,306027979,155421,183747359,29591001,1 +20201119 03:00,1,20200930,1493,286615446,42718510932,142418590,306030279,155421,184285427,29591001,1 +20201120 03:00,1,20200930,1499,286696224,42730610558,142418590,306030479,155421,184285427,29591001,1 +20201121 03:00,1,20200331,1465,294835429,33701926901,105016517,276471475,84822,209150925,31507932,1 +20201126 03:00,1,20200930,1502,286727605,42735311558,142418590,306030479,155421,184295188,29591001,1 +20201128 03:00,1,20200930,1503,286754447,42739332558,142418590,306030479,155421,184295188,29591001,1 +20201201 03:00,1,20200930,1506,287324631,42824741558,142418590,306030479,155421,184832981,29591795,1 +20201202 03:00,2,20200630,1523,296521179,42261602601,163309923,274619253,258484,200044339,23736139,1,20200930,1507,287333222,42826028558,142418590,306030479,155421,184836877,29591795,1 +20201204 03:00,1,20200930,1508,287370068,42831547558,142418590,306030479,155421,184873723,29591795,1 +20201210 03:00,1,20200930,1509,287377659,42832684558,142418590,306030479,155421,184873723,29591795,1 +20201211 03:00,1,20200930,1510,287420976,42839172558,142418590,306030479,155421,184917040,29591795,1 +20201212 03:00,1,20200930,1511,287424872,42839756558,142418590,306030479,155421,184920936,29591795,1 +20201215 03:00,1,20200930,1512,287429573,42840460558,142418590,306030479,155421,184924196,29591795,1 +20201219 03:00,1,20200331,1466,294880426,33707076901,105016517,276471475,84822,209195922,31507932,1 +20201222 03:00,1,20200930,1513,287454299,42844164558,142418590,306030479,155421,184948922,29591795,1 +20201223 03:00,1,20200930,1514,287457891,42844702558,142418590,306030479,155421,184952514,29591795,1 +20201224 03:00,2,20200331,1467,294883220,33707396901,105016517,276471475,84822,209195922,31507932,1,20200630,1525,296565022,42267879601,163309923,274619253,258484,200083712,23736139,1 +20201230 03:00,1,20200930,1515,287496788,42850528558,142418590,306030479,155421,184991411,29591795,1 diff --git a/output/alternative/sec/13f/qqq.csv b/output/alternative/sec/13f/qqq.csv new file mode 100644 index 0000000..2408bf2 --- /dev/null +++ b/output/alternative/sec/13f/qqq.csv @@ -0,0 +1,50 @@ +20201002 03:00,1,20200930,6,78694,21864000,0,0,0,3942,0,0 +20201003 03:00,2,20161231,876,135250828,14221552744,65775900,124845911,4342970,74450430,21587107,1,20200930,11,98635,27406000,0,0,0,10424,0,1 +20201006 03:00,1,20200930,15,124869,33947000,0,0,0,34122,0,1 +20201007 03:00,3,20170331,925,131167733,16390626665,78949302,127816453,84470,80642722,21158945,1,20170930,989,151571898,22034738042,102711417,236894149,72571,91704080,27994046,1,20200930,35,354965,97878775,0,0,0,118681,0,1 +20201008 03:00,4,20170630,1000,186674444,25683114791,117992818,292256044,986742,133949358,24746192,1,20200331,1552,201602380,36370294998,126432758,174206098,34522,93606199,22975417,1,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,48,671179,185265775,0,0,0,157974,13830,1 +20201009 03:00,1,20200930,67,1655515,458743775,0,0,0,1043729,13830,1 +20201010 03:00,1,20200930,90,7471116,2074607775,20,356,0,6298303,13830,1 +20201014 03:00,3,20171231,1134,154184093,23885976507,92193508,247789533,57807,99186325,23109895,1,20180331,1156,160165531,25152579909,120245995,339842133,96927,100391981,27578394,1,20200930,154,9193255,2549033775,20,356,0,6823107,33684,1 +20201015 03:00,1,20200930,189,10117931,2806045563,20,1356,0,7550422,35289,1 +20201016 03:00,1,20200930,217,10912367,3026829563,20,3856,0,7832181,52618,1 +20201017 03:00,1,20200930,249,12050974,3343634563,3020,16556,0,8419759,56910,1 +20201020 03:00,1,20200930,292,13322868,3696527563,5329,344493,0,8921100,60124,1 +20201021 03:00,1,20200930,327,13734686,3811066563,5329,474693,0,9034171,60124,1 +20201022 03:00,1,20200930,359,14660295,4068238563,5329,475693,0,9380415,60124,1 +20201023 03:00,1,20200930,384,15304206,4253132563,5329,475693,0,9728801,60124,1 +20201024 03:00,2,20181231,1210,189347348,26778021216,120984467,204669094,70581,102868825,22053436,1,20200930,439,16945607,4709331379,5329,511141,0,10198647,62374,1 +20201027 03:00,1,20200930,494,17690019,4914001379,133729,1247141,4548,10596983,71154,1 +20201028 03:00,1,20200930,540,22275080,6187805379,1748329,2192441,4548,14136407,72720,1 +20201029 03:00,1,20200930,573,26686591,7413666379,1785129,2228441,4548,18367764,73190,1 +20201030 03:00,1,20200930,618,29093956,8082047379,1785129,2278441,4548,19107917,73281,1 +20201031 03:00,1,20200930,667,45070866,12522615990,1791079,2317256,12719,20251732,14350429,1 +20201103 03:00,2,20200630,1650,207831174,51823138977,107813897,213108396,28651,105480853,20820134,1,20200930,708,46628476,12948843991,2311179,2318256,43349,20491782,14351782,1 +20201104 03:00,1,20200930,770,49135108,13645200011,2320679,2475756,52481,21847688,14351790,1 +20201105 03:00,1,20200930,809,57615366,16001171641,2320679,2518756,52481,25815836,14351790,1 +20201106 03:00,2,20200630,1651,207865091,51831536977,107813897,213108396,28651,105514770,20820134,1,20200930,862,66967156,18598004485,8064379,8394336,52481,32451409,14357695,1 +20201107 03:00,2,20200630,1652,207867759,51832197977,107813897,213108396,28651,105517438,20820134,1,20200930,909,77307218,21471054452,12420089,9039336,52481,38029517,17897141,1 +20201110 03:00,1,20200930,968,87206225,24219899650,12425089,9049436,52481,39778485,17897241,1 +20201111 03:00,2,20181231,1211,189347348,26778021216,120984467,204669094,70636,102868825,22053436,1,20200930,1051,97858813,27182652406,12458674,9111772,52481,48464023,17902640,1 +20201113 03:00,7,20181231,1212,189373987,26782130216,120984467,204669094,70636,102872247,22053436,1,20190331,1275,159473833,28564707293,171831006,226859415,89814,79914370,24352133,1,20190630,1301,183740052,30198186088,132752435,206937829,42894,88691984,18747698,1,20190930,1327,162338950,30837665565,118672842,195144317,24570,89345942,19357851,1,20191231,1533,172544390,36339910470,107562626,197177432,32027,93375221,14943082,1,20200331,1553,201634217,36376356998,126432758,174206098,34522,93611121,22975417,1,20200930,1253,134374027,37174400464,53536874,67471669,52481,71355937,21946887,1 +20201114 03:00,3,20191231,1534,172548925,36340874470,107562626,197177432,32027,93375221,14943082,1,20200630,1653,207874248,51833804977,107813897,213108396,28651,105517438,20820134,1,20200930,1469,207577593,57505342391,99268474,153644269,62672,89852176,30658566,1 +20201117 03:00,3,20200331,1554,201638752,36377219998,126432758,174206098,34522,93611121,22975417,1,20200630,1654,207878783,51834927977,107813897,213108396,28651,105517438,20820134,1,20200930,1676,227026493,62907381395,170578174,294426569,104443,103418203,31976473,1 +20201118 03:00,1,20200930,1692,228482161,63312129395,178814774,312438469,104443,104375799,32345398,1 +20201119 03:00,1,20200930,1695,228488734,63313955395,178814774,312438969,104443,104379584,32345398,1 +20201120 03:00,1,20200930,1700,228733797,63382044415,178815074,312522269,104443,104381874,32345398,1 +20201121 03:00,1,20200930,1701,228733902,63382072415,178815074,312522269,104443,104381874,32345398,1 +20201125 03:00,1,20200331,1555,201987713,36444246998,126608358,174986798,34522,93960082,22975417,1 +20201126 03:00,1,20200930,1703,228743065,63384618415,178815074,312522269,104443,104383506,32345398,1 +20201128 03:00,1,20200930,1706,228811099,63403701415,178815074,312522269,104443,104384875,32345398,1 +20201201 03:00,1,20200930,1709,229111347,63487184415,178815074,312522269,104443,104487213,32345398,1 +20201202 03:00,1,20200930,1711,229118427,63489146415,178815074,312522269,104443,104494077,32345398,1 +20201204 03:00,2,20191231,1535,172556032,36342385470,107562626,197177432,32027,93375221,14943082,1,20200930,1714,229219625,63517263415,178815074,312522269,104443,104501069,32345398,1 +20201208 03:00,1,20200930,1715,229221042,63517666415,178815074,312522269,104443,104501152,32345398,1 +20201210 03:00,1,20200930,1716,229222135,63517970415,178815074,312522269,104443,104501152,32345398,1 +20201211 03:00,1,20200930,1718,229303236,63540503415,178815074,312522269,104443,104582253,32345398,1 +20201215 03:00,1,20200930,1721,229310785,63542600415,178815074,312562269,104443,104584030,32345398,1 +20201219 03:00,2,20200331,1556,202039282,36454064998,126608358,174986798,34522,94011651,22975417,1,20200930,1722,229341896,63551573415,178815074,312562269,104443,104584030,32345398,1 +20201222 03:00,1,20200930,1723,229343256,63551951415,178815074,312562269,104443,104585390,32345398,1 +20201223 03:00,1,20200930,1724,229347747,63553199415,178815074,312562269,104443,104589881,32345398,1 +20201224 03:00,1,20200630,1655,207928796,51847309977,107813897,213108396,28651,105567451,20820134,1 +20201230 03:00,1,20200930,1725,229397349,63566980415,178815074,312562269,104443,104639483,32345398,1 diff --git a/output/alternative/sec/13f/spy.csv b/output/alternative/sec/13f/spy.csv new file mode 100644 index 0000000..747f1dd --- /dev/null +++ b/output/alternative/sec/13f/spy.csv @@ -0,0 +1,52 @@ +20201002 03:00,1,20200930,7,83319,27883000,0,0,0,50566,0,0 +20201003 03:00,1,20200930,12,185055,61953000,0,0,0,130705,0,1 +20201006 03:00,1,20200930,23,342285,114606000,0,0,0,219995,595,1 +20201007 03:00,1,20200930,48,2799209,937407041,0,700,0,1216812,595,1 +20201008 03:00,3,20200331,2455,745478677,175509591048,424659174,576382812,523144,515399393,69326981,1,20200630,2446,584341986,183639959549,369139121,542631260,313306,389643779,40614362,1,20200930,65,3333215,1116190041,0,700,0,1285424,2536,1 +20201009 03:00,1,20200930,98,4531571,1517505310,0,800,0,2375391,5209,1 +20201010 03:00,1,20200930,130,9091756,3044675310,1,1922,0,6254637,5376,1 +20201014 03:00,5,20171231,2003,703727322,187718983509,320775258,641793785,829991,742821309,78880513,1,20180930,2031,768103910,190882152567,398525325,581978132,610967,634176622,99556009,1,20200331,2456,745478677,175509591048,424659174,576382812,523486,515399393,69326981,1,20200630,2447,584347013,183641602549,369139121,542631260,313306,389643779,40614362,1,20200930,211,11446011,3833638310,22101,26809,0,7476623,9716,1 +20201015 03:00,1,20200930,264,12274438,4111035562,30601,30309,0,7950475,40950,1 +20201016 03:00,1,20200930,305,13348859,4470888562,30601,30309,0,8553850,63154,1 +20201017 03:00,2,20191231,2506,640052698,206785547550,298970465,474608407,411653,485823096,39737037,1,20200930,350,14181068,4750156562,575401,700509,0,8987206,63154,1 +20201020 03:00,1,20200930,407,15311390,5128110562,13593308,7701054,0,9553807,65182,1 +20201021 03:00,2,20200630,2448,584347786,183641840549,369139121,542631260,313306,389644552,40614362,1,20200930,455,17108431,5730078562,13722708,7831354,0,10440965,65182,1 +20201022 03:00,1,20200930,504,18287139,6050883782,13722708,8290954,0,11315731,66782,1 +20201023 03:00,1,20200930,547,20481207,6774629411,13722708,8290954,0,12783684,66782,1 +20201024 03:00,4,20181231,2278,791620859,170490428836,359424966,6695261836,643965,506324660,85496398,1,20191231,2507,640053673,206785861550,298970465,474608407,411653,485823096,39737037,1,20200331,2457,745479752,175509868048,424659174,576382812,523486,515399393,69326981,1,20200930,613,22174619,7342248470,13922708,8316851,0,13603092,137677,1 +20201027 03:00,1,20200930,686,28494551,9426203528,14806108,12284795,88694,17074888,143844,1 +20201028 03:00,2,20200630,2449,584351217,183642898549,369139121,542631260,313306,389644552,40614362,1,20200930,752,35984200,11934460528,15000208,12358995,88694,23763898,144264,1 +20201029 03:00,1,20200930,807,39182698,13005878528,69398708,21102095,88694,26413531,236893,1 +20201030 03:00,1,20200930,864,43353486,14402299528,69398708,26862695,111966,28963145,248807,1 +20201031 03:00,1,20200930,927,52079658,17306259078,69670758,28293545,374715,30912598,6033690,1 +20201103 03:00,7,20181231,2279,791624300,170491288836,359424966,6695261836,643965,506324660,85496398,1,20190331,2223,621746923,174693340246,287143357,477629432,418324,442734112,71841838,1,20190630,2241,602703559,177301291734,319568319,514267687,407485,426318551,61238098,1,20190930,2243,621533686,182726161999,310369399,510602077,367726,653808050,64435135,1,20191231,2508,640057119,206786970550,298970465,474608407,411653,485823096,39737037,1,20200331,2458,745483181,175510752048,424659174,576382812,523486,515399393,69326981,1,20200930,992,54960350,18264320404,69670862,30564147,410457,31651870,6034028,1 +20201104 03:00,1,20200930,1077,58883599,19578844598,69704962,30825847,416849,32608140,6058887,1 +20201105 03:00,1,20200930,1140,71571977,24427557208,70067662,32335147,416849,38650623,7925096,1 +20201106 03:00,2,20200630,2450,584357824,183644935549,369139121,542631260,313306,389651159,40614362,1,20200930,1219,90674603,30818705904,89052662,36000566,416849,51494813,8025414,1 +20201107 03:00,2,20200630,2451,584361679,183646124549,369139121,542631260,313306,389655013,40614362,1,20200930,1298,104158528,35306444143,112396762,37202674,416849,63221211,8994575,1 +20201110 03:00,1,20200930,1396,118457318,40070237542,112476862,37446774,416849,67964381,9010671,1 +20201111 03:00,2,20181231,2280,791624300,170491288836,359424966,6695261836,644307,506324660,85496398,1,20200930,1510,155734138,52565215724,113193667,38961213,417191,104294589,9036597,1 +20201113 03:00,7,20181231,2281,791629915,170492691836,359424966,6695261836,644307,506324660,85496398,1,20190331,2224,621752384,174694883246,287143357,477629432,418324,442734112,71841838,1,20190630,2242,602708915,177302860734,319568319,514267687,407485,426318551,61238098,1,20190930,2244,621539042,182727750999,310369399,510602077,367726,653808050,64435135,1,20191231,2509,640063638,206789068550,298970465,474608407,411653,485823096,39737037,1,20200331,2459,745489796,175512457048,424659174,576382812,523486,515399393,69326981,1,20200930,1809,321090647,107671284571,180233167,138499031,417191,242271683,16768236,1 +20201114 03:00,3,20191231,2510,640084797,206795878550,298970465,474608407,411653,485823096,39737037,1,20200630,2452,584402935,183658846549,369139121,542631260,313306,389655013,40614362,1,20200930,2113,493974287,168282315902,226976017,264348981,417709,306260662,37625938,1 +20201117 03:00,3,20200331,2460,745510955,175517911048,424659174,576382812,523486,515399393,69326981,1,20200630,2454,584424223,183665410549,369139121,542631260,313306,389655013,40614362,1,20200930,2404,559277340,195050512921,375436630,561053620,543662,360730366,41483299,1 +20201118 03:00,1,20200930,2426,564913916,196939092921,391247330,567820720,543662,365825915,41754254,1 +20201119 03:00,4,20191231,2511,640116053,206805938550,298970465,474608407,411653,485823096,39737037,1,20200331,2461,745540372,175525493048,424659174,576382812,523486,515399393,69326981,1,20200630,2455,584453029,183674293549,369139121,542631260,313306,389655013,40614362,1,20200930,2432,565090921,196998376921,391913330,568098620,543662,365927152,41754254,1 +20201120 03:00,1,20200930,2442,565479991,197128699599,391915330,568101920,543662,366127201,41754254,1 +20201121 03:00,1,20200930,2443,565480119,197128741599,391915330,568101920,543662,366127201,41754254,1 +20201124 03:00,2,20200331,2462,745541606,175525811048,424659174,576382812,523486,515399393,69326981,1,20200930,2444,565481974,197129362599,391915330,568101920,543662,366127201,41754254,1 +20201125 03:00,1,20200930,2445,565485875,197130668599,391915330,568101920,543662,366130858,41754254,1 +20201126 03:00,1,20200930,2448,565505691,197137305599,391915330,568101920,543662,366141374,41754254,1 +20201128 03:00,1,20200930,2449,565595759,197161598599,391915330,568101920,543662,366141374,41754254,1 +20201201 03:00,3,20200331,2463,745569396,175532974048,424659174,576382812,523486,515399393,69326981,1,20200630,2456,584483230,183683606549,369139121,542631260,313306,389655013,40614362,1,20200930,2454,565972279,197287799599,391915330,568101920,543662,366352055,41754440,1 +20201202 03:00,2,20200630,2457,584572721,183686917549,369139121,542631260,313306,389655013,40614362,1,20200930,2456,566076852,197322822599,391915330,568101920,543662,366418141,41754440,1 +20201203 03:00,1,20200930,2457,566155427,197349136599,391915330,568101920,543662,366496716,41754440,1 +20201204 03:00,2,20191231,2512,640143993,206814931550,298970465,474608407,411653,485823096,39737037,1,20200930,2461,566546411,197480073599,391915330,568101920,543662,366662160,41754440,1 +20201208 03:00,1,20200930,2462,566548934,197480939599,391915330,568101920,543662,366662744,41754440,1 +20201211 03:00,1,20200930,2464,566727744,197540820599,391915330,568101920,543662,366841554,41754440,1 +20201212 03:00,1,20200930,2465,566736187,197543648599,391915330,568101920,543662,366849547,41754440,1 +20201215 03:00,1,20200930,2469,566741967,197545582599,391925330,568142820,543662,366851599,41754440,1 +20201219 03:00,2,20200331,2464,745642755,175551881048,424659174,576382812,523486,515472752,69326981,1,20200930,2470,566749043,197548082599,391937430,568142820,543662,366851599,41754440,1 +20201222 03:00,1,20200930,2471,566883706,197593179599,391937430,568142820,543662,366986262,41754440,1 +20201224 03:00,2,20200630,2459,584648016,183710135549,369139121,542631260,313306,389727287,40614362,1,20200930,2473,566887134,197594327599,391937430,568142820,543662,366986262,41754440,1 +20201230 03:00,1,20200930,2474,566953969,197616708599,391937430,568142820,543662,367053097,41754440,1 +20201231 03:00,4,20190331,2225,621752384,174694883246,287143357,477629432,418666,442734112,71841838,1,20190630,2243,602708915,177302860734,319568319,514267687,407827,426318551,61238098,1,20190930,2245,621539042,182727750999,310369399,510602077,368068,653808050,64435135,1,20200630,2460,584648016,183710135549,369139121,542631260,313648,389727287,40614362,1 diff --git a/output/alternative/sec/13f/universe/20201002.csv b/output/alternative/sec/13f/universe/20201002.csv new file mode 100644 index 0000000..bf4e8ae --- /dev/null +++ b/output/alternative/sec/13f/universe/20201002.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3322,2593727943,930994271399.059,62056050,60809616,9515,1617776740,171210197,1,20200930,11,4091893,473319000,0,0,0,3104369,320,0 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,1,125,3000,0,0,0,125,0,0 +BNO UN3IMQ2JU1YD,BNO,1,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0 +EEM SNQLASP67O85,EEM,2,20200630,1093,402002172,16085435604,162866492,266937301,941231,298463240,30555320,1,20200930,4,75204,3315000,0,0,0,67226,0,0 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2577,224500855,311707313305,3153704,4007761,864,144997911,12179536,1,20200930,8,39809,58503000,0,0,0,18575,19,0 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2809,233451052,327023246860,5357763,5656871,5140,152205269,18129403,1,20200930,4,55446,81261000,0,0,0,41379,0,0 +IBM R735QTJ8XC9X,IBM,2,20200630,1946,506086831,60863188152,10227584,12215460,32374,302618181,43346758,1,20200930,8,148094,18019000,0,0,0,83301,0,0 +IWM RV0PWMLXVHPH,IWM,2,20200630,1519,296176353,42212844601,163309923,274619253,258484,200042574,23736139,1,20200930,3,10158,1521000,0,0,0,3900,0,0 +NB R735QTJ8XC9X,BAC,2,20200630,2091,5920295025,141114307534.821,107384568,157707032,187805,4003729388,536801725,1,20200930,6,756603,18260000,0,0,0,571542,0,0 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1648,207726673,51328479977,107557897,212857396,28651,105377120,20820134,1,20200930,6,78694,21864000,0,0,0,3942,0,0 +SPY R735QTJ8XC9X,SPY,2,20200630,2445,584333533,183637353549,369139121,542631260,313306,389636810,40614362,1,20200930,7,83319,27883000,0,0,0,50566,0,0 +USO THORT68ZZSYT,USO,1,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1 +UW R735QTJ8XC9X,WM,2,20200630,1323,304450944,32194956671,1541045,1920563,53618,198442222,19384570,1,20200930,5,162449,18383000,0,0,0,150636,0,0 diff --git a/output/alternative/sec/13f/universe/20201005.csv b/output/alternative/sec/13f/universe/20201005.csv new file mode 100644 index 0000000..6675e74 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201005.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3322,2593727943,930994271399.059,62056050,60809616,9515,1617776740,171210197,1,20200930,18,4958860,573723000,0,0,0,3878678,320,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,1,125,3000,0,0,0,125,0,0 +BNO UN3IMQ2JU1YD,BNO,1,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0 +EEM SNQLASP67O85,EEM,2,20200630,1093,402002172,16085435604,162866492,266937301,941231,298463240,30555320,1,20200930,5,242449,10689000,0,0,0,217775,0,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2577,224500855,311707313305,3153704,4007761,864,144997911,12179536,1,20200930,15,66166,97237000,0,0,0,31431,19,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2809,233451052,327023246860,5357763,5656871,5140,152205269,18129403,1,20200930,12,72383,106086000,0,0,0,54170,0,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1946,506086831,60863188152,10227584,12215460,32374,302618181,43346758,1,20200930,12,177831,21638000,0,0,0,104638,0,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1519,296176353,42212844601,163309923,274619253,258484,200042574,23736139,1,20200930,6,264998,39694000,0,0,0,253677,0,1 +NB R735QTJ8XC9X,BAC,2,20200630,2091,5920295025,141114307534.821,107384568,157707032,187805,4003729388,536801725,1,20200930,9,1029191,24827000,0,0,0,830166,0,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1648,207726673,51328479977,107557897,212857396,28651,105377120,20820134,1,20200930,11,98635,27406000,0,0,0,10424,0,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2445,584333533,183637353549,369139121,542631260,313306,389636810,40614362,1,20200930,12,185055,61953000,0,0,0,130705,0,1 +USO THORT68ZZSYT,USO,1,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1 +UW R735QTJ8XC9X,WM,2,20200630,1323,304450944,32194956671,1541045,1920563,53618,198442222,19384570,1,20200930,6,180919,20474000,0,0,0,169069,0,0 diff --git a/output/alternative/sec/13f/universe/20201006.csv b/output/alternative/sec/13f/universe/20201006.csv new file mode 100644 index 0000000..6306bf3 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201006.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3322,2593727943,930994271399.059,62056050,60809616,9515,1617776740,171210197,1,20200930,35,9689407,1123082000,0,0,0,8174627,1880,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,2,199450,5490000,0,0,0,199450,0,0 +BNO UN3IMQ2JU1YD,BNO,1,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0 +EEM SNQLASP67O85,EEM,2,20200630,1093,402002172,16085435604,162866492,266937301,941231,298463240,30555320,1,20200930,13,3308578,145875000,0,0,0,3192434,0,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2577,224500855,311707313305,3153704,4007761,864,144997911,12179536,1,20200930,25,163841,240678000,0,0,0,129004,19,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2809,233451052,327023246860,5357763,5656871,5140,152205269,18129403,1,20200930,23,148893,218219000,0,0,0,130595,0,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1946,506086831,60863188152,10227584,12215460,32374,302618181,43346758,1,20200930,23,434335,53051000,0,0,0,299645,1052,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1519,296176353,42212844601,163309923,274619253,258484,200042574,23736139,1,20200930,13,401407,60127000,0,0,0,365776,0,1 +NB R735QTJ8XC9X,BAC,2,20200630,2091,5920295025,141114307534.821,107384568,157707032,187805,4003729388,536801725,1,20200930,18,2543323,61347000,0,0,0,2338057,0,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1648,207726673,51328479977,107557897,212857396,28651,105377120,20820134,1,20200930,15,124869,33947000,0,0,0,34122,0,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2445,584333533,183637353549,369139121,542631260,313306,389636810,40614362,1,20200930,23,342285,114606000,0,0,0,219995,595,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,1,37,1000,0,0,0,0,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1323,304450944,32194956671,1541045,1920563,53618,198442222,19384570,1,20200930,13,368332,41682000,0,0,0,308597,1385,0 diff --git a/output/alternative/sec/13f/universe/20201007.csv b/output/alternative/sec/13f/universe/20201007.csv new file mode 100644 index 0000000..5938f8c --- /dev/null +++ b/output/alternative/sec/13f/universe/20201007.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3322,2593727943,930994271399.059,62056050,60809616,9515,1617776740,171210197,1,20200930,69,14907129,1750280233,0,160,0,11581134,3768,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,8,253571,6979000,0,0,0,212840,0,0 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,1,940,10000,0,0,0,940,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1093,402002172,16085435604,162866492,266937301,941231,298463240,30555320,1,20200930,25,3497442,154200000,0,0,0,3323223,0,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2577,224500855,311707313305,3153704,4007761,864,144997911,12179536,1,20200930,46,248706,365044000,0,0,0,182194,39,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2809,233451052,327023246860,5357763,5656871,5140,152205269,18129403,1,20200930,49,232394,340531080,0,0,0,202023,61,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1946,506086831,60863188152,10227584,12215460,32374,302618181,43346758,1,20200930,45,752019,91733544,0,0,0,454324,1186,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1519,296176353,42212844601,163309923,274619253,258484,200042574,23736139,1,20200930,31,701402,105086712,0,0,0,395122,430,1 +NB R735QTJ8XC9X,BAC,2,20200630,2091,5920295025,141114307534.821,107384568,157707032,187805,4003729388,536801725,1,20200930,37,5958568,143562578,0,0,0,4734425,365,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1648,207726673,51328479977,107557897,212857396,28651,105377120,20820134,1,20200930,35,354965,97878775,0,0,0,118681,0,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2445,584333533,183637353549,369139121,542631260,313306,389636810,40614362,1,20200930,48,2799209,937407041,0,700,0,1216812,595,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,5,4406,124000,0,0,0,773,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1323,304450944,32194956671,1541045,1920563,53618,198442222,19384570,1,20200930,33,609798,69007000,0,0,0,361654,1385,0 diff --git a/output/alternative/sec/13f/universe/20201008.csv b/output/alternative/sec/13f/universe/20201008.csv new file mode 100644 index 0000000..2df87f1 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201008.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3323,2593821891,931028543399.059,62056050,60809616,9515,1617836453,171210197,1,20200930,95,17230133,2018852233,0,160,0,13057035,17606,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,10,253584,6979000,0,0,0,212853,0,0 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,1,940,10000,0,0,0,940,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1094,402227786,16094457604,162866492,266937301,941231,298688854,30555320,1,20200930,34,3703189,163258000,0,0,0,3360744,2808,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2578,224501641,311708424305,3153704,4007761,864,144998537,12179536,1,20200930,57,491270,721519000,0,0,0,420408,139,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,64,614898,901129080,0,0,0,573539,73,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1947,506089972,60863567152,10227584,12215460,32374,302621172,43346758,1,20200930,57,823509,100425544,0,0,0,483097,3957,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1519,296176353,42212844601,163309923,274619253,258484,200042574,23736139,1,20200930,42,809407,121291712,0,0,0,447795,2238,1 +NB R735QTJ8XC9X,BAC,2,20200630,2092,5920326659,141115058534.821,107384568,157707032,187805,4003759522,536801725,1,20200930,49,8431054,203698578,0,0,0,7070918,6643,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,48,671179,185265775,0,0,0,157974,13830,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2446,584341986,183639959549,369139121,542631260,313306,389643779,40614362,1,20200930,65,3333215,1116190041,0,700,0,1285424,2536,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,6,4435,125000,0,0,0,802,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,40,669387,75751000,0,0,0,416004,4354,0 diff --git a/output/alternative/sec/13f/universe/20201009.csv b/output/alternative/sec/13f/universe/20201009.csv new file mode 100644 index 0000000..4011e28 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201009.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3323,2593821891,931028543399.059,62056050,60809616,9515,1617836453,171210197,1,20200930,137,25809440,3012204066,0,160,0,19079235,143648,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,17,814025,22408000,0,0,0,761142,4064,0 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,3,112419,1196888,0,0,0,112391,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1094,402227786,16094457604,162866492,266937301,941231,298688854,30555320,1,20200930,43,4124808,181846798,0,0,0,3763761,2808,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2578,224501641,311708424305,3153704,4007761,864,144998537,12179536,1,20200930,87,655988,963589080,0,0,0,532405,3020,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,99,861799,1263114502,0,0,0,777642,1347,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1947,506089972,60863567152,10227584,12215460,32374,302621172,43346758,1,20200930,84,1304767,158980544,0,0,0,788482,8973,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1519,296176353,42212844601,163309923,274619253,258484,200042574,23736139,1,20200930,61,1252939,187729824,0,0,0,868908,2238,1 +NB R735QTJ8XC9X,BAC,2,20200630,2092,5920326659,141115058534.821,107384568,157707032,187805,4003759522,536801725,1,20200930,74,12398039,299330382,0,0,0,9946257,44175,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,67,1655515,458743775,0,0,0,1043729,13830,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2446,584341986,183639959549,369139121,542631260,313306,389643779,40614362,1,20200930,98,4531571,1517505310,0,800,0,2375391,5209,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,8,11815,334000,0,0,0,7936,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,59,1111072,125738071,0,0,0,753362,25118,0 diff --git a/output/alternative/sec/13f/universe/20201012.csv b/output/alternative/sec/13f/universe/20201012.csv new file mode 100644 index 0000000..8337d26 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201012.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3323,2593821891,931028543399.059,62056050,60809616,9515,1617836453,171210197,1,20200930,176,39295067,4573620066,400,560,0,28711663,172403,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,22,1863033,51286000,0,0,0,1740936,4064,0 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,3,112419,1196888,0,0,0,112391,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1094,402227786,16094457604,162866492,266937301,941231,298688854,30555320,1,20200930,55,7271037,320562798,0,100,0,6620726,2808,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2578,224501641,311708424305,3153704,4007761,864,144998537,12179536,1,20200930,119,923050,1356033080,0,0,0,725848,3853,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,137,1184258,1735711502,0,0,0,1038573,1580,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1947,506089972,60863567152,10227584,12215460,32374,302621172,43346758,1,20200930,116,2157748,262781544,1,0,0,1322040,9801,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1519,296176353,42212844601,163309923,274619253,258484,200042574,23736139,1,20200930,75,4494500,673280824,0,0,0,3897875,2238,1 +NB R735QTJ8XC9X,BAC,2,20200630,2092,5920326659,141115058534.821,107384568,157707032,187805,4003759522,536801725,1,20200930,95,17885579,436429382,0,0,0,14661589,52000,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,90,7471116,2074607775,20,356,0,6298303,13830,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2446,584341986,183639959549,369139121,542631260,313306,389643779,40614362,1,20200930,130,9091756,3044675310,1,1922,0,6254637,5376,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,11,36427,1030000,450,0,0,28081,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,75,1825671,206609071,0,0,0,1355389,30268,0 diff --git a/output/alternative/sec/13f/universe/20201013.csv b/output/alternative/sec/13f/universe/20201013.csv new file mode 100644 index 0000000..8337d26 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201013.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3323,2593821891,931028543399.059,62056050,60809616,9515,1617836453,171210197,1,20200930,176,39295067,4573620066,400,560,0,28711663,172403,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,22,1863033,51286000,0,0,0,1740936,4064,0 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,3,112419,1196888,0,0,0,112391,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1094,402227786,16094457604,162866492,266937301,941231,298688854,30555320,1,20200930,55,7271037,320562798,0,100,0,6620726,2808,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2578,224501641,311708424305,3153704,4007761,864,144998537,12179536,1,20200930,119,923050,1356033080,0,0,0,725848,3853,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,137,1184258,1735711502,0,0,0,1038573,1580,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1947,506089972,60863567152,10227584,12215460,32374,302621172,43346758,1,20200930,116,2157748,262781544,1,0,0,1322040,9801,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1519,296176353,42212844601,163309923,274619253,258484,200042574,23736139,1,20200930,75,4494500,673280824,0,0,0,3897875,2238,1 +NB R735QTJ8XC9X,BAC,2,20200630,2092,5920326659,141115058534.821,107384568,157707032,187805,4003759522,536801725,1,20200930,95,17885579,436429382,0,0,0,14661589,52000,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,90,7471116,2074607775,20,356,0,6298303,13830,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2446,584341986,183639959549,369139121,542631260,313306,389643779,40614362,1,20200930,130,9091756,3044675310,1,1922,0,6254637,5376,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,11,36427,1030000,450,0,0,28081,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,75,1825671,206609071,0,0,0,1355389,30268,0 diff --git a/output/alternative/sec/13f/universe/20201014.csv b/output/alternative/sec/13f/universe/20201014.csv new file mode 100644 index 0000000..95da4b2 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201014.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3323,2593821891,931028543399.059,62056050,60809616,9515,1617836453,171210197,1,20200930,292,56867940,6605513066,13200,560,0,38735805,367467,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,33,1874445,51601000,532200,1108100,0,1750192,4064,0 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,3,112419,1196888,0,0,0,112391,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1095,402262479,16095967604,162866492,266937301,941231,298688854,30555320,1,20200930,82,8584156,378460798,0,100,0,7522690,17199,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2578,224501641,311708424305,3153704,4007761,864,144998537,12179536,1,20200930,194,1146839,1685005080,1600,0,0,845543,12861,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,221,1718755,2519254502,0,0,0,1545929,12625,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,178,3091325,376468544,1,0,0,1947058,11467,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,119,7224789,1082332824,0,0,0,5698818,169054,1 +NB R735QTJ8XC9X,BAC,2,20200630,2092,5920326659,141115058534.821,107384568,157707032,187805,4003759522,536801725,1,20200930,164,26881097,653377382,30700,0,0,20315317,414778,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,154,9193255,2549033775,20,356,0,6823107,33684,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2447,584347013,183641602549,369139121,542631260,313306,389643779,40614362,1,20200930,211,11446011,3833638310,22101,26809,0,7476623,9716,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,17,83368,2357000,450,0,0,49643,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,117,2856384,323540071,0,0,0,2086826,30268,1 diff --git a/output/alternative/sec/13f/universe/20201015.csv b/output/alternative/sec/13f/universe/20201015.csv new file mode 100644 index 0000000..ae6793a --- /dev/null +++ b/output/alternative/sec/13f/universe/20201015.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3323,2593821891,931028543399.059,62056050,60809616,9515,1617836453,171210197,1,20200930,361,75989294,8810423071,14800,3360,0,46762952,5814007,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,42,2237144,61587663,532200,1108100,0,1966135,106315,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,4,141670,1507888,0,0,0,141642,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1095,402262479,16095967604,162866492,266937301,941231,298688854,30555320,1,20200930,104,10102847,445425798,5000,100,0,8714631,17199,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2578,224501641,311708424305,3153704,4007761,864,144998537,12179536,1,20200930,237,1396842,2052388826,1600,0,0,1005680,62587,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,272,2260723,3307265083,0,0,0,1685849,362483,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,214,3667122,446506052,1,0,0,2222289,136014,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,149,9197080,1377715000,0,0,0,7401226,169054,1 +NB R735QTJ8XC9X,BAC,2,20200630,2092,5920326659,141115058534.821,107384568,157707032,187805,4003759522,536801725,1,20200930,203,38920543,943389448,34200,0,0,22216474,9450660,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,189,10117931,2806045563,20,1356,0,7550422,35289,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2447,584347013,183641602549,369139121,542631260,313306,389643779,40614362,1,20200930,264,12274438,4111035562,30601,30309,0,7950475,40950,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,21,439203,12423000,450,0,0,404754,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,144,3639389,412152311,0,0,53096,2692685,129248,1 diff --git a/output/alternative/sec/13f/universe/20201016.csv b/output/alternative/sec/13f/universe/20201016.csv new file mode 100644 index 0000000..f903a9a --- /dev/null +++ b/output/alternative/sec/13f/universe/20201016.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3323,2593821891,931028543399.059,62056050,60809616,9515,1617836453,171210197,1,20200930,412,87565867,10152043071,14800,3360,0,55685812,6477369,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,50,2352876,64773663,532200,1108100,0,2026286,106315,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,4,141670,1507888,0,0,0,141642,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1095,402262479,16095967604,162866492,266937301,941231,298688854,30555320,1,20200930,119,12409155,547301798,5000,100,0,10532580,39782,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2578,224501641,311708424305,3153704,4007761,864,144998537,12179536,1,20200930,276,1645069,2417205412,1600,0,0,1157940,149291,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,311,2751843,4027110083,0,0,0,2006533,489612,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,241,4131114,502976052,1,0,0,2591105,136164,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,171,9904049,1483616000,0,0,0,7935522,172763,1 +NB R735QTJ8XC9X,BAC,2,20200630,2092,5920326659,141115058534.821,107384568,157707032,187805,4003759522,536801725,1,20200930,230,43126619,1045477448,34200,0,0,25717236,9504159,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,217,10912367,3026829563,20,3856,0,7832181,52618,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2447,584347013,183641602549,369139121,542631260,313306,389643779,40614362,1,20200930,305,13348859,4470888562,30601,30309,0,8553850,63154,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,24,464083,13128000,450,0,0,405804,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,162,4734636,536191311,0,0,53096,3647684,131227,1 diff --git a/output/alternative/sec/13f/universe/20201019.csv b/output/alternative/sec/13f/universe/20201019.csv new file mode 100644 index 0000000..16a3705 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201019.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3323,2593821891,931028543399.059,62056050,60809616,9515,1617836453,171210197,1,20200930,474,97168833,11361440573,14800,162960,0,61340763,6511738,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,57,2687186,73976663,532200,1108100,0,2240609,106315,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,4,141670,1507888,0,0,0,141642,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1095,402262479,16095967604,162866492,266937301,941231,298688854,30555320,1,20200930,138,13066953,576414798,78900,144900,0,10649854,39782,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2578,224501641,311708424305,3153704,4007761,864,144998537,12179536,1,20200930,320,1813996,2665347412,1600,400,0,1264449,149511,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,359,2925622,4280764083,0,1800,0,2140774,490989,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,270,4707297,573067052,1,0,0,2991007,139851,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,199,11426506,1711960000,45600,24000,0,8398019,176674,1 +NB R735QTJ8XC9X,BAC,2,20200630,2092,5920326659,141115058534.821,107384568,157707032,187805,4003759522,536801725,1,20200930,266,48863061,1189252270,34200,11900,0,28807403,9524297,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,249,12050974,3343634563,3020,16556,0,8419759,56910,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2447,584347013,183641602549,369139121,542631260,313306,389643779,40614362,1,20200930,350,14181068,4750156562,575401,700509,0,8987206,63154,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,25,464681,13145000,450,0,0,405804,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,183,5185078,585922311,0,1000,53096,3939434,137785,1 diff --git a/output/alternative/sec/13f/universe/20201020.csv b/output/alternative/sec/13f/universe/20201020.csv new file mode 100644 index 0000000..56615de --- /dev/null +++ b/output/alternative/sec/13f/universe/20201020.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3323,2593821891,931028543399.059,62056050,60809616,9515,1617836453,171210197,1,20200930,562,120716866,14088930573,157063,242434,0,80266579,6607992,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,74,3527041,97098663,532200,1109654,0,2675574,106565,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,4,141670,1507888,0,0,0,141642,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1095,402262479,16095967604,162866492,266937301,941231,298688854,30555320,1,20200930,160,13505916,595772798,78900,144900,0,10962497,39782,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2578,224501641,311708424305,3153704,4007761,864,144998537,12179536,1,20200930,390,2461129,3616362412,1610,400,0,1818993,150924,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,431,4467195,6539167083,10500,1800,0,3602077,491501,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,314,6092234,741566052,73,197,0,3816802,149285,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,230,11766925,1762950000,47024,25094,0,8591641,176975,1 +NB R735QTJ8XC9X,BAC,2,20200630,2092,5920326659,141115058534.821,107384568,157707032,187805,4003759522,536801725,1,20200930,317,61882346,1502914270,34200,11900,0,38577651,9557415,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,292,13322868,3696527563,5329,344493,0,8921100,60124,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2447,584347013,183641602549,369139121,542631260,313306,389643779,40614362,1,20200930,407,15311390,5128110562,13593308,7701054,0,9553807,65182,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,29,472805,13374000,32250,120,0,413832,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,211,6517398,736693311,0,1000,53096,5019001,139722,1 diff --git a/output/alternative/sec/13f/universe/20201021.csv b/output/alternative/sec/13f/universe/20201021.csv new file mode 100644 index 0000000..953677b --- /dev/null +++ b/output/alternative/sec/13f/universe/20201021.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3324,2593823376,931029085399.059,62056050,60809616,9515,1617837938,171210197,1,20200930,639,157484024,18347940573,161063,642182,0,112233684,6607992,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,90,4268692,117514663,532200,1131341,0,3382047,106565,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,4,141670,1507888,0,0,0,141642,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1095,402262479,16095967604,162866492,266937301,941231,298688854,30555320,1,20200930,184,30776151,1357213798,78900,144900,0,27817219,39782,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2579,224586503,311828423305,3153704,4007761,864,145083399,12179536,1,20200930,443,3160050,4642606412,1610,7556,0,2449520,150924,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,492,5329928,7803748083,10900,7675,0,4402043,491501,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,356,8553412,1041040052,73,22059,0,6037148,149285,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,261,12881970,1929974000,47024,25094,0,8844839,176975,1 +NB R735QTJ8XC9X,BAC,2,20200630,2093,5921695548,141147549534.821,107384568,157707032,187805,4005128411,536801725,1,20200930,360,88438227,2142912270,34200,197330,0,64222447,9557415,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,327,13734686,3811066563,5329,474693,0,9034171,60124,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2448,584347786,183641840549,369139121,542631260,313306,389644552,40614362,1,20200930,455,17108431,5730078562,13722708,7831354,0,10440965,65182,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,32,473590,13396000,32250,120,0,413832,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,238,8087201,914346311,0,36044,53096,6424636,139722,1 diff --git a/output/alternative/sec/13f/universe/20201022.csv b/output/alternative/sec/13f/universe/20201022.csv new file mode 100644 index 0000000..eb67901 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201022.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3324,2593823376,931029085399.059,62056050,60809616,9515,1617837938,171210197,1,20200930,705,181930015,21141618592,165063,784882,0,134381394,6625583,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,101,5252476,144597663,532200,1131341,0,4364869,106565,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,4,141670,1507888,0,0,0,141642,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1095,402262479,16095967604,162866492,266937301,941231,298688854,30555320,1,20200930,202,31294089,1377641883,78900,144900,0,28188813,40207,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2579,224586503,311828423305,3153704,4007761,864,145083399,12179536,1,20200930,489,3592913,5278700412,1610,7556,0,2846527,151111,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,545,6126127,8970702083,10900,7675,0,5116428,492161,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,398,10055303,1214978052,4273,22059,0,7284722,161373,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,297,14603766,2029839000,47024,25094,0,10304633,176975,1 +NB R735QTJ8XC9X,BAC,2,20200630,2093,5921695548,141147549534.821,107384568,157707032,187805,4005128411,536801725,1,20200930,402,109122823,2637786270,136800,197330,0,83444326,9644813,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,359,14660295,4068238563,5329,475693,0,9380415,60124,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2448,584347786,183641840549,369139121,542631260,313306,389644552,40614362,1,20200930,504,18287139,6050883782,13722708,8290954,0,11315731,66782,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,35,482492,13648000,32250,120,0,421580,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,266,9075043,1026143311,0,36044,59780,7313293,142722,1 diff --git a/output/alternative/sec/13f/universe/20201023.csv b/output/alternative/sec/13f/universe/20201023.csv new file mode 100644 index 0000000..f594497 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201023.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3325,2593825742,931029948399.059,62056050,60809616,9515,1617839777,171210197,1,20200930,766,196351669,22803325261,165063,884882,0,144301015,6625583,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,109,5711252,157228663,532200,1131341,0,4638827,106565,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,5,152720,1625888,0,0,0,152692,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1095,402262479,16095967604,162866492,266937301,941231,298688854,30555320,1,20200930,220,32815323,1444712883,78900,144900,0,29202316,40207,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2579,224586503,311828423305,3153704,4007761,864,145083399,12179536,1,20200930,531,3857137,5667178412,1610,7556,0,3020398,151111,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,589,6384926,9350745012,10900,7675,0,5324010,492161,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,430,10481806,1267036052,4273,22059,0,7485854,161373,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,322,16144149,2260812432,47024,25094,0,11379444,176975,1 +NB R735QTJ8XC9X,BAC,2,20200630,2093,5921695548,141147549534.821,107384568,157707032,187805,4005128411,536801725,1,20200930,432,116005754,2803623907,141800,197330,0,89544309,9644813,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,384,15304206,4253132563,5329,475693,0,9728801,60124,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2448,584347786,183641840549,369139121,542631260,313306,389644552,40614362,1,20200930,547,20481207,6774629411,13722708,8290954,0,12783684,66782,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,38,510085,14429000,32250,120,0,437866,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,283,9485921,1072658311,0,36044,59780,7526650,142722,1 diff --git a/output/alternative/sec/13f/universe/20201026.csv b/output/alternative/sec/13f/universe/20201026.csv new file mode 100644 index 0000000..a8a8420 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201026.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3325,2593825742,931029948399.059,62056050,60809616,9515,1617839777,171210197,1,20200930,848,208002125,24154113644,170063,1087798,0,151474348,6789146,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,117,6280671,172902663,532200,1131341,0,5144698,106565,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,5,152720,1625888,0,0,0,152692,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1095,402262479,16095967604,162866492,266937301,941231,298688854,30555320,1,20200930,246,33444576,1472473883,878900,144900,0,29588429,40207,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2579,224586503,311828423305,3153704,4007761,864,145083399,12179536,1,20200930,585,4113133,6043500092,1610,8436,0,3229291,154184,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,646,6650093,9740280816,10900,8934,0,5542366,498329,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,469,14175100,1716306623,4273,26339,0,9658494,172175,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,351,16835333,2364361432,47024,35048,0,11790739,265014,1 +NB R735QTJ8XC9X,BAC,2,20200630,2093,5921695548,141147549534.821,107384568,157707032,187805,4005128411,536801725,1,20200930,470,141627185,3420883402,141800,214351,0,109334322,9687522,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,439,16945607,4709331379,5329,511141,0,10198647,62374,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2448,584347786,183641840549,369139121,542631260,313306,389644552,40614362,1,20200930,613,22174619,7342248470,13922708,8316851,0,13603092,137677,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,39,510710,14447000,32250,120,0,438491,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,303,10903524,1233085311,0,37857,59780,8320874,142722,1 diff --git a/output/alternative/sec/13f/universe/20201027.csv b/output/alternative/sec/13f/universe/20201027.csv new file mode 100644 index 0000000..bfdde73 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201027.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3325,2593825742,931029948399.059,62056050,60809616,9515,1617839777,171210197,1,20200930,940,237603684,27585584434,419488,1414923,179897,176260922,7186046,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,131,8072630,222234939,532200,1131341,0,6922800,106565,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,5,152720,1625888,0,0,0,152692,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1095,402262479,16095967604,162866492,266937301,941231,298688854,30555320,1,20200930,279,34313850,1509227292,2195900,161600,8063,29972928,53957,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2579,224586503,311828423305,3153704,4007761,864,145083399,12179536,1,20200930,657,4584703,6737058274,1610,9836,1378,3660754,171511,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,722,7356397,10774933938,32162,13496,418,6187982,500381,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,523,15662443,1897313306,107173,105539,14141,10963081,177897,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,397,17620529,2482042626,523324,144848,19850,12104866,265230,1 +NB R735QTJ8XC9X,BAC,2,20200630,2093,5921695548,141147549534.821,107384568,157707032,187805,4005128411,536801725,1,20200930,530,155588398,3758546927,1177000,397951,58812,122192023,9718057,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,494,17690019,4914001379,133729,1247141,4548,10596983,71154,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2448,584347786,183641840549,369139121,542631260,313306,389644552,40614362,1,20200930,686,28494551,9426203528,14806108,12284795,88694,17074888,143844,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,44,757021,21415000,36750,120,0,684658,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,336,11717079,1325164921,0,39057,59906,9076919,142722,1 diff --git a/output/alternative/sec/13f/universe/20201028.csv b/output/alternative/sec/13f/universe/20201028.csv new file mode 100644 index 0000000..f5f98ae --- /dev/null +++ b/output/alternative/sec/13f/universe/20201028.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3326,2593827856,931030719399.059,62056050,60809616,9515,1617839777,171210197,1,20200930,1024,262000144,30450716434,1198888,1832723,179897,195580088,7230642,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,142,9096963,252398939,569400,1153241,0,7773972,106565,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,6,152720,1625888,10100,3600,0,152692,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1095,402262479,16095967604,162866492,266937301,941231,298688854,30555320,1,20200930,305,35458793,1559657292,3144700,2044000,8063,30582085,53957,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2579,224586503,311828423305,3153704,4007761,864,145083399,12179536,1,20200930,718,5237393,7690343274,5510,11736,1378,4214064,172162,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,793,7630943,11171654938,35862,22996,418,6407786,500504,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,572,16881356,2047206306,166773,112939,14141,11913440,179797,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,429,18463120,2608237626,1225624,996748,19850,12605749,265232,1 +NB R735QTJ8XC9X,BAC,2,20200630,2093,5921695548,141147549534.821,107384568,157707032,187805,4005128411,536801725,1,20200930,580,166927747,4034986927,1835400,851051,58812,130565585,9737617,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,540,22275080,6187805379,1748329,2192441,4548,14136407,72720,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2449,584351217,183642898549,369139121,542631260,313306,389644552,40614362,1,20200930,752,35984200,11934460528,15000208,12358995,88694,23763898,144264,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,49,856032,24215000,2632750,1434420,0,780031,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,372,12342844,1395887921,25100,50657,59906,9503136,142897,1 diff --git a/output/alternative/sec/13f/universe/20201029.csv b/output/alternative/sec/13f/universe/20201029.csv new file mode 100644 index 0000000..f26115c --- /dev/null +++ b/output/alternative/sec/13f/universe/20201029.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3326,2593827856,931030719399.059,62056050,60809616,9515,1617839777,171210197,1,20200930,1102,281292371,32685377434,1423688,2266523,179897,206129245,13478469,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,149,9342804,259167939,569400,1153241,0,7806941,316630,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,6,152720,1625888,10100,3600,0,152692,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1095,402262479,16095967604,162866492,266937301,941231,298688854,30555320,1,20200930,326,36827580,1620005292,3144700,2044000,8063,31497296,56746,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2579,224586503,311828423305,3153704,4007761,864,145083399,12179536,1,20200930,771,5638175,8279094274,9110,11736,1378,4402980,351518,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,852,8033133,11761241938,40362,46196,418,6657749,612388,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,606,18037575,2187906306,166773,112939,14141,12481487,474656,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,456,19078232,2700376626,1225624,1007348,19850,12875834,265652,1 +NB R735QTJ8XC9X,BAC,2,20200630,2093,5921695548,141147549534.821,107384568,157707032,187805,4005128411,536801725,1,20200930,622,173534550,4194184927,1863900,892851,58812,133844364,12490340,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,573,26686591,7413666379,1785129,2228441,4548,18367764,73190,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2449,584351217,183642898549,369139121,542631260,313306,389644552,40614362,1,20200930,807,39182698,13005878528,69398708,21102095,88694,26413531,236893,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,52,856911,24240000,2689750,1434420,0,780642,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,404,13215888,1494709921,25100,50657,59906,9912443,425065,1 diff --git a/output/alternative/sec/13f/universe/20201030.csv b/output/alternative/sec/13f/universe/20201030.csv new file mode 100644 index 0000000..53ae08f --- /dev/null +++ b/output/alternative/sec/13f/universe/20201030.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3326,2593827856,931030719399.059,62056050,60809616,9515,1617839777,171210197,1,20200930,1187,335219945,38915282434,1435688,2298123,179897,240020571,14222551,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,160,10112339,280305939,569400,1153241,0,8330129,339372,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,8,411573,4380888,10100,3600,0,162067,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1095,402262479,16095967604,162866492,266937301,941231,298688854,30555320,1,20200930,355,44167941,1943657292,3144700,2044000,8063,37589953,112180,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2579,224586503,311828423305,3153704,4007761,864,145083399,12179536,1,20200930,839,6312860,9269540274,9110,12336,1378,4905197,359282,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,920,9419513,13791223335,40362,46196,418,7681486,621897,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,660,22725288,2757410306,166773,112939,14141,15397644,507407,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,496,27328552,3919458626,1225624,1007348,19850,20636875,284463,1 +NB R735QTJ8XC9X,BAC,2,20200630,2093,5921695548,141147549534.821,107384568,157707032,187805,4005128411,536801725,1,20200930,676,195312881,4718025927,1863900,892851,58812,148634565,12618239,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,618,29093956,8082047379,1785129,2278441,4548,19107917,73281,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2449,584351217,183642898549,369139121,542631260,313306,389644552,40614362,1,20200930,864,43353486,14402299528,69398708,26862695,111966,28963145,248807,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,55,980643,27741000,2689750,1434420,0,806626,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,435,15921640,1801223921,25100,50657,59906,11320844,434224,1 diff --git a/output/alternative/sec/13f/universe/20201102.csv b/output/alternative/sec/13f/universe/20201102.csv new file mode 100644 index 0000000..c90645b --- /dev/null +++ b/output/alternative/sec/13f/universe/20201102.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3326,2593827856,931030719399.059,62056050,60809616,9515,1617839777,171210197,1,20200930,1279,502482474,58326680208,1484162,2441468,179897,381248549,32227185,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,181,36409265,1005440691,575012,1164438,0,31331321,1715278,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,8,411573,4380888,10100,3600,0,162067,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1095,402262479,16095967604,162866492,266937301,941231,298688854,30555320,1,20200930,395,52503296,2312065481,3163944,2078809,388547,42598084,2655622,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2579,224586503,311828423305,3153704,4007761,864,145083399,12179536,1,20200930,910,10010129,14713986654,10331,13343,1378,8190475,632294,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,995,12620305,18492827100,40788,47915,418,10351214,912339,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,720,31889412,3870733925,169725,122219,14141,23165399,1391740,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,542,37124553,5389247894,1226632,1028514,90100,22039115,7997034,1 +NB R735QTJ8XC9X,BAC,2,20200630,2093,5921695548,141147549534.821,107384568,157707032,187805,4005128411,536801725,1,20200930,734,267350927,6422501015,1979165,1011640,58812,212317433,17955248,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1649,207731917,51329777977,107557897,212857396,28651,105381596,20820134,1,20200930,667,45070866,12522615990,1791079,2317256,12719,20251732,14350429,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2449,584351217,183642898549,369139121,542631260,313306,389644552,40614362,1,20200930,927,52079658,17306259078,69670758,28293545,374715,30912598,6033690,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,59,987093,27923000,2689750,1434420,416,806626,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,475,19388951,2194015344,25145,51495,59906,14419184,630177,1 diff --git a/output/alternative/sec/13f/universe/20201103.csv b/output/alternative/sec/13f/universe/20201103.csv new file mode 100644 index 0000000..ff36d22 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201103.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3327,2593836160,931097500399.059,62128750,60820416,9515,1617848081,171210197,1,20200930,1371,542538359,63024209872,1880781,2441493,323322,405686028,32906196,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,193,36786721,1015922691,575012,1164438,0,31678026,1719343,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,9,490973,5226888,10100,3600,0,241467,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1095,402262479,16095967604,162866492,266937301,941231,298688854,30555320,1,20200930,414,53656506,2362870481,3164344,2108709,388547,43227369,2655622,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2579,224586503,311828423305,3153704,4007761,864,145083399,12179536,1,20200930,981,11508876,16912314953,10331,13343,1378,9180479,648243,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,1076,13488765,19761846541,40788,47915,4026,10950984,916433,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,778,33672083,4086225926,170225,122232,14141,24518596,1399455,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,579,38025744,5511358907,1226732,1580814,90100,22367693,8000335,1 +NB R735QTJ8XC9X,BAC,2,20200630,2094,5921794705,141149904534.821,107384568,157832032,187805,4005227568,536801725,1,20200930,793,285109858,6850103015,1979165,1011640,198346,225553033,18416145,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1650,207831174,51823138977,107813897,213108396,28651,105480853,20820134,1,20200930,708,46628476,12948843991,2311179,2318256,43349,20491782,14351782,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2449,584351217,183642898549,369139121,542631260,313306,389644552,40614362,1,20200930,992,54960350,18264320404,69670862,30564147,410457,31651870,6034028,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,62,1005774,28451000,2689750,1434420,416,807384,0,0 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,506,20535980,2323788344,25145,51495,59906,15043659,632064,1 diff --git a/output/alternative/sec/13f/universe/20201104.csv b/output/alternative/sec/13f/universe/20201104.csv new file mode 100644 index 0000000..0fae6d3 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201104.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3327,2593836160,931097500399.059,62128750,60820416,9515,1617848081,171210197,1,20200930,1495,583656879,67796108182,1897781,2487993,323322,429468870,35483834,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,209,38162470,1053798691,575012,1164438,0,33021993,1729142,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,11,493897,5257888,10100,3600,0,241467,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1095,402262479,16095967604,162866492,266937301,941231,298688854,30555320,1,20200930,458,57503743,2532495348,3172344,2116709,395147,43864367,2698842,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2579,224586503,311828423305,3153704,4007761,864,145083399,12179536,1,20200930,1072,12375745,18186244328,10331,13343,1378,9744835,726095,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,1179,14993874,21967601002,40788,47915,4026,11959590,1140973,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,852,35273301,4271845971,170225,122232,14141,25649825,1407630,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,638,39615389,5749440908,2076032,1819214,90100,22989258,8019610,1 +NB R735QTJ8XC9X,BAC,2,20200630,2094,5921794705,141149904534.821,107384568,157832032,187805,4005227568,536801725,1,20200930,866,300534503,7221669939,2025165,1017640,198346,233887842,19901723,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1650,207831174,51823138977,107813897,213108396,28651,105480853,20820134,1,20200930,770,49135108,13645200011,2320679,2475756,52481,21847688,14351790,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2449,584351217,183642898549,369139121,542631260,313306,389644552,40614362,1,20200930,1077,58883599,19578844598,69704962,30825847,416849,32608140,6058887,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,68,1056817,29896000,2709750,1434420,416,810592,0,1 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,554,22453888,2540670928,25145,51495,59906,16363793,766253,1 diff --git a/output/alternative/sec/13f/universe/20201105.csv b/output/alternative/sec/13f/universe/20201105.csv new file mode 100644 index 0000000..fe2c0b0 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201105.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3327,2593836160,931097500399.059,62128750,60820416,9515,1617848081,171210197,1,20200930,1585,721657922,83755142343,2305881,2517493,323322,524962745,41161027,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,222,41869159,1141833691,575012,1164438,0,36403992,1888196,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,13,3790183,39839888,10100,3600,0,275768,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1095,402262479,16095967604,162866492,266937301,941231,298688854,30555320,1,20200930,488,60587466,2668470464,3246644,2123709,395147,44986379,2710198,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2579,224586503,311828423305,3153704,4007761,864,145083399,12179536,1,20200930,1134,14821286,21790313774,10331,13343,1378,11716664,819985,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,1253,17984427,26351193487,40788,47915,4026,14335219,1252829,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,906,42599141,5162938156,187325,122232,14141,31016865,1544414,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,675,42176620,6133086908,2088832,1839214,90100,24170364,8098044,1 +NB R735QTJ8XC9X,BAC,2,20200630,2094,5921794705,141149904534.821,107384568,157832032,187805,4005227568,536801725,1,20200930,918,334528957,8039722179,2073665,1017640,198346,254998365,21960050,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1650,207831174,51823138977,107813897,213108396,28651,105480853,20820134,1,20200930,809,57615366,16001171641,2320679,2518756,52481,25815836,14351790,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2449,584351217,183642898549,369139121,542631260,313306,389644552,40614362,1,20200930,1140,71571977,24427557208,70067662,32335147,416849,38650623,7925096,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,72,2287581,64271000,2742150,1434420,416,844922,0,1 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,587,25288258,2861474928,25145,51495,59906,18243958,831910,1 diff --git a/output/alternative/sec/13f/universe/20201106.csv b/output/alternative/sec/13f/universe/20201106.csv new file mode 100644 index 0000000..5dd5352 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201106.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3329,2593846426,931101245399.059,62128750,60820416,9515,1617853675,171210197,1,20200930,1690,931150080,108071204326,4821281,5704841,323322,695459972,41733069,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,243,53232267,1454764691,575012,1164438,0,44927033,3244651,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,15,3987561,41940888,10100,3600,0,441164,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1096,402276414,16096524604,162866492,266937301,941231,298688854,30555320,1,20200930,525,79579772,3505837834,8802244,8538905,395147,59990446,2716815,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2580,224586777,311828810305,3153704,4007761,864,145083673,12179536,1,20200930,1221,19001593,27933888098,10331,13543,1378,15347468,833230,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2810,233454986,327028825860,5357763,5656871,5140,152206396,18129403,1,20200930,1344,23668795,34681915324,59588,48915,4026,18958287,1262897,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1948,506122832,60867795152,10227584,12215460,32374,302621172,43346758,1,20200930,971,58332004,7077118156,205625,190832,14141,45149476,1564231,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1520,296187000,42214421601,163309923,274619253,258484,200042574,23736139,1,20200930,730,70268332,10340973652,5697832,11870318,90100,50031589,9272715,1 +NB R735QTJ8XC9X,BAC,2,20200630,2094,5921794705,141149904534.821,107384568,157832032,187805,4005227568,536801725,1,20200930,986,417578801,10040517660,2366365,1859040,198346,326554050,21974910,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1651,207865091,51831536977,107813897,213108396,28651,105514770,20820134,1,20200930,862,66967156,18598004485,8064379,8394336,52481,32451409,14357695,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2450,584357824,183644935549,369139121,542631260,313306,389651159,40614362,1,20200930,1219,90674603,30818705904,89052662,36000566,416849,51494813,8025414,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,78,2532546,71200000,2742150,1434420,416,990815,0,1 +UW R735QTJ8XC9X,WM,2,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,625,38638823,4372188928,25145,51495,59906,30447487,857380,1 diff --git a/output/alternative/sec/13f/universe/20201109.csv b/output/alternative/sec/13f/universe/20201109.csv new file mode 100644 index 0000000..954f63e --- /dev/null +++ b/output/alternative/sec/13f/universe/20201109.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3330,2593998975,931156895399.059,62128750,60820416,9515,1618006224,171210197,1,20200930,1794,2186512123,250840711020,6839831,6549741,323322,1751081437,62844626,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,271,136656426,3719285691,575038,1164438,0,120111870,4074531,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,15,3987561,41940888,10100,3600,0,441164,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1096,402276414,16096524604,162866492,266937301,941231,298688854,30555320,1,20200930,563,159398006,7026027341,10982744,8538920,395147,138221250,3383968,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2581,224587604,311829979305,3153704,4007761,864,145084500,12179536,1,20200930,1302,42712028,62161229912,10336,13543,1378,35201649,1193874,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2811,233475674,327058162860,5357763,5656871,5140,152227084,18129403,1,20200930,1432,48183674,69998358829,109689,48915,4026,39364575,1639975,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1949,506133620,60869098152,10227584,12215460,32374,302631960,43346758,1,20200930,1043,129771803,15619935965,206673,190876,14141,103730857,3248883,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1521,296188765,42214674601,163309923,274619253,258484,200044339,23736139,1,20200930,781,93602526,13836541492,5897840,13293721,90100,71265796,10241510,1 +NB R735QTJ8XC9X,BAC,2,20200630,2095,5922353148,141163167534.821,107387568,157832032,187805,4005786010,536801725,1,20200930,1053,1006929903,23969974708,11376585,1859040,198346,825776842,30187580,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1652,207867759,51832197977,107813897,213108396,28651,105517438,20820134,1,20200930,909,77307218,21471054452,12420089,9039336,52481,38029517,17897141,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2451,584361679,183646124549,369139121,542631260,313306,389655013,40614362,1,20200930,1298,104158528,35306444143,112396762,37202674,416849,63221211,8994575,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,87,2573265,72351000,2742650,1434420,416,1001213,0,1 +UW R735QTJ8XC9X,WM,2,20200630,1325,304456491,32195543671,1541045,1920563,53618,198447769,19384570,1,20200930,668,74783369,8410098241,25145,51495,59906,61856436,1450480,1 diff --git a/output/alternative/sec/13f/universe/20201110.csv b/output/alternative/sec/13f/universe/20201110.csv new file mode 100644 index 0000000..4ea1e1b --- /dev/null +++ b/output/alternative/sec/13f/universe/20201110.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3330,2593998975,931156895399.059,62128750,60820416,9515,1618006224,171210197,1,20200930,1942,2413493728,277125289352,7107031,6938841,323322,1915747491,67944880,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,307,188850591,5225975691,663838,1214438,0,163476393,4145177,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,16,4039083,42489888,10100,3600,0,441164,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1096,402276414,16096524604,162866492,266937301,941231,298688854,30555320,1,20200930,611,174296779,7682964827,10984544,8541720,395147,146185308,3421847,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2581,224587604,311829979305,3153704,4007761,864,145084500,12179536,1,20200930,1426,47695125,69490051903,10336,13543,1378,39104506,1213678,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2811,233475674,327058162860,5357763,5656871,5140,152227084,18129403,1,20200930,1553,54442424,79155715322,109689,59215,4026,44189362,1802598,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1949,506133620,60869098152,10227584,12215460,32374,302631960,43346758,1,20200930,1125,144724228,17439563354,206673,190876,14141,112316531,3624605,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1521,296188765,42214674601,163309923,274619253,258484,200044339,23736139,1,20200930,839,101739943,15056316861,5923690,13300321,90100,74885241,10274675,1 +NB R735QTJ8XC9X,BAC,2,20200630,2095,5922353148,141163167534.821,107387568,157832032,187805,4005786010,536801725,1,20200930,1149,1143385115,27244754208,12470485,1859040,198346,930613068,34150221,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1652,207867759,51832197977,107813897,213108396,28651,105517438,20820134,1,20200930,968,87206225,24219899650,12425089,9049436,52481,39778485,17897241,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2451,584361679,183646124549,369139121,542631260,313306,389655013,40614362,1,20200930,1396,118457318,40070237542,112476862,37446774,416849,67964381,9010671,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,93,3372023,94949000,2751150,1434420,416,1397449,0,1 +UW R735QTJ8XC9X,WM,2,20200630,1325,304456491,32195543671,1541045,1920563,53618,198447769,19384570,1,20200930,728,85129921,9581386241,25145,51495,59906,67534975,1797595,1 diff --git a/output/alternative/sec/13f/universe/20201111.csv b/output/alternative/sec/13f/universe/20201111.csv new file mode 100644 index 0000000..67a36ea --- /dev/null +++ b/output/alternative/sec/13f/universe/20201111.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3330,2593998975,931156895399.059,62128750,60820416,9515,1618006224,171210197,1,20200930,2105,3238509099,372680064632,7147831,6956785,323322,2623557761,84837625,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,331,231804858,6408482425,663838,1214438,0,201654104,4691194,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,16,4039083,42489888,10100,3600,0,441164,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1096,402276414,16096524604,162866492,266937301,941231,298688854,30555320,1,20200930,676,189056625,8333895191,10985396,8660843,395147,159274840,3654000,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2581,224587604,311829979305,3153704,4007761,864,145084500,12179536,1,20200930,1555,62436874,91154786589,10336,13543,1434,51527443,1555747,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2811,233475674,327058162860,5357763,5656871,5140,152227084,18129403,1,20200930,1692,70696311,102979660549,112289,59218,4237,63348499,2102442,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1949,506133620,60869098152,10227584,12215460,32374,302631960,43346758,1,20200930,1214,201961196,24403463612,206673,200876,38428,151692949,12139456,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1521,296188765,42214674601,163309923,274619253,258484,200044339,23736139,1,20200930,916,107194763,15873277195,5939690,13383377,90100,88241958,10330204,1 +NB R735QTJ8XC9X,BAC,2,20200630,2095,5922353148,141163167534.821,107387568,157832032,187805,4005786010,536801725,1,20200930,1237,1510749367,36094972525,12476425,1859618,368177,1254808865,39602911,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1652,207867759,51832197977,107813897,213108396,28651,105517438,20820134,1,20200930,1051,97858813,27182652406,12458674,9111772,52481,48464023,17902640,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2451,584361679,183646124549,369139121,542631260,313306,389655013,40614362,1,20200930,1510,155734138,52565215724,113193667,38961213,417191,104294589,9036597,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,101,3421906,96358590,2751150,1434420,416,1408231,0,1 +UW R735QTJ8XC9X,WM,2,20200630,1325,304456491,32195543671,1541045,1920563,53618,198447769,19384570,1,20200930,790,108785914,12258610204,25145,51495,60206,86200372,4190674,1 diff --git a/output/alternative/sec/13f/universe/20201112.csv b/output/alternative/sec/13f/universe/20201112.csv new file mode 100644 index 0000000..67a36ea --- /dev/null +++ b/output/alternative/sec/13f/universe/20201112.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3330,2593998975,931156895399.059,62128750,60820416,9515,1618006224,171210197,1,20200930,2105,3238509099,372680064632,7147831,6956785,323322,2623557761,84837625,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,331,231804858,6408482425,663838,1214438,0,201654104,4691194,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,16,4039083,42489888,10100,3600,0,441164,0,0 +EEM SNQLASP67O85,EEM,2,20200630,1096,402276414,16096524604,162866492,266937301,941231,298688854,30555320,1,20200930,676,189056625,8333895191,10985396,8660843,395147,159274840,3654000,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2581,224587604,311829979305,3153704,4007761,864,145084500,12179536,1,20200930,1555,62436874,91154786589,10336,13543,1434,51527443,1555747,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2811,233475674,327058162860,5357763,5656871,5140,152227084,18129403,1,20200930,1692,70696311,102979660549,112289,59218,4237,63348499,2102442,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1949,506133620,60869098152,10227584,12215460,32374,302631960,43346758,1,20200930,1214,201961196,24403463612,206673,200876,38428,151692949,12139456,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1521,296188765,42214674601,163309923,274619253,258484,200044339,23736139,1,20200930,916,107194763,15873277195,5939690,13383377,90100,88241958,10330204,1 +NB R735QTJ8XC9X,BAC,2,20200630,2095,5922353148,141163167534.821,107387568,157832032,187805,4005786010,536801725,1,20200930,1237,1510749367,36094972525,12476425,1859618,368177,1254808865,39602911,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1652,207867759,51832197977,107813897,213108396,28651,105517438,20820134,1,20200930,1051,97858813,27182652406,12458674,9111772,52481,48464023,17902640,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2451,584361679,183646124549,369139121,542631260,313306,389655013,40614362,1,20200930,1510,155734138,52565215724,113193667,38961213,417191,104294589,9036597,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,101,3421906,96358590,2751150,1434420,416,1408231,0,1 +UW R735QTJ8XC9X,WM,2,20200630,1325,304456491,32195543671,1541045,1920563,53618,198447769,19384570,1,20200930,790,108785914,12258610204,25145,51495,60206,86200372,4190674,1 diff --git a/output/alternative/sec/13f/universe/20201113.csv b/output/alternative/sec/13f/universe/20201113.csv new file mode 100644 index 0000000..3971c50 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201113.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,2,20200630,3330,2593998975,931156895399.059,62128750,60820416,9515,1618006224,171210197,1,20200930,2529,4422426158,508083527582,39479891,49472685,323322,3551215918,131071704,1 +AIG R735QTJ8XC9X,AIG,2,20200630,676,768179661,23961077747,9363225,12100518,0,492064415,74555899,1,20200930,432,322527890,8905164425,1033738,3324738,0,276178379,7458154,1 +BNO UN3IMQ2JU1YD,BNO,2,20200630,46,9148101,99072832,341600,375300,0,5053228,30000,0,20200930,24,4938986,52054888,10100,3600,0,1286839,7948,0 +EEM SNQLASP67O85,EEM,2,20200630,1096,402276414,16096524604,162866492,266937301,941231,298688854,30555320,1,20200930,824,266533266,11736518249,86036396,65758343,395147,214685104,14565259,1 +GOOCV VP83T1ZUHROL,GOOG,2,20200630,2581,224587604,311829979305,3153704,4007761,864,145084500,12179536,1,20200930,1903,99422358,145587926836,1497984,603289,1434,80305160,2904420,1 +GOOG T1AZ164W5VTX,GOOGL,2,20200630,2811,233475674,327058162860,5357763,5656871,5140,152227084,18129403,1,20200930,2068,112086118,155107805246,1087289,846218,11053,97633690,3404534,1 +IBM R735QTJ8XC9X,IBM,2,20200630,1949,506133620,60869098152,10227584,12215460,32374,302631960,43346758,1,20200930,1476,267828177,32393190765,902373,1544276,38428,203713089,14260527,1 +IWM RV0PWMLXVHPH,IWM,2,20200630,1521,296188765,42214674601,163309923,274619253,258484,200044339,23736139,1,20200930,1097,162300377,24123452741,31753390,92463033,92269,130510320,16755092,1 +NB R735QTJ8XC9X,BAC,2,20200630,2095,5922353148,141163167534.821,107387568,157832032,187805,4005786010,536801725,1,20200930,1508,2255076839,53914098779,29795975,37626718,368177,1886887938,58611709,1 +QQQ RIWIV7K5Z9LX,QQQ,2,20200630,1652,207867759,51832197977,107813897,213108396,28651,105517438,20820134,1,20200930,1253,134374027,37174400464,53536874,67471669,52481,71355937,21946887,1 +SPY R735QTJ8XC9X,SPY,2,20200630,2451,584361679,183646124549,369139121,542631260,313306,389655013,40614362,1,20200930,1809,321090647,107671284571,180233167,138499031,417191,242271683,16768236,1 +USO THORT68ZZSYT,USO,2,20200630,216,31388051,880062518,37951028,62688308,446,26813779,879630,1,20200930,130,6223988,174622278,4662310,1941548,416,3482183,584302,1 +UW R735QTJ8XC9X,WM,2,20200630,1325,304456491,32195543671,1541045,1920563,53618,198447769,19384570,1,20200930,986,154286920,17376098914,128745,277795,60206,120266607,5326730,1 diff --git a/output/alternative/sec/13f/universe/20201116.csv b/output/alternative/sec/13f/universe/20201116.csv new file mode 100644 index 0000000..0a3ec2a --- /dev/null +++ b/output/alternative/sec/13f/universe/20201116.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,2945,6173804333,710865538773,93151391,132427133,352074,4608041427,181671946,1 +AIG R735QTJ8XC9X,AIG,1,20200930,542,490285207,13523471982,5759568,8947138,0,397453915,24752304,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,32,5090083,53659888,147100,87900,0,1348813,7948,0 +EEM SNQLASP67O85,EEM,1,20200930,983,377110091,16597398369,134371996,220280743,961076,278668597,27473637,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2260,147782941,216526221953,3824884,2117989,1434,114669405,4503653,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2443,171109889,238726546549,2292889,2272918,11926,135958289,5157054,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1723,368808059,44667481135,3826473,6089976,38557,275925073,17966388,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1295,264730189,39438609932,71609190,184060004,92269,165275688,28534807,1 +NB R735QTJ8XC9X,BAC,1,20200930,1774,3289520295,78828283847,75736775,75098718,368233,2619818718,89143852,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1469,207577593,57505342391,99268474,153644269,62672,89852176,30658566,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2113,493974287,168282315902,226976017,264348981,417709,306260662,37625938,1 +USO THORT68ZZSYT,USO,1,20200930,158,12551111,353323410,8822434,8868420,416,8889941,717430,1 +UW R735QTJ8XC9X,WM,1,20200930,1164,216353749,24393874616,367845,542295,60206,166423211,8794925,1 diff --git a/output/alternative/sec/13f/universe/20201117.csv b/output/alternative/sec/13f/universe/20201117.csv new file mode 100644 index 0000000..46d080b --- /dev/null +++ b/output/alternative/sec/13f/universe/20201117.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3274,9663010703,1114962933259,265228991,291501233,649898,6140539968,640717603,1 +AIG R735QTJ8XC9X,AIG,1,20200930,628,762838163,20974206289,10173268,17334438,0,491284427,72384594,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,37,7893886,83519888,230500,232700,0,4152516,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1100,417251323,18366585369,165745296,273437743,961076,311124798,32950852,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2538,215033575,315369159923,9144704,5588289,1434,140109348,12185080,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2758,235083685,332102492865,7436089,7149018,11926,156818072,18774703,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1894,494513673,59960517738,9657073,12334776,46773,299507638,40577520,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1473,285448404,42543352932,134723990,299100279,155421,183312946,29508001,1 +NB R735QTJ8XC9X,BAC,1,20200930,1994,5862651638,140806127242,121131275,153617618,1388986,4038714740,484979627,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1676,227026493,62907381395,170578174,294426569,104443,103418203,31976473,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2404,559277340,195050512921,375436630,561053620,543662,360730366,41483299,1 +USO THORT68ZZSYT,USO,1,20200930,179,33782626,953980410,22853426,38616768,69466,30100605,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1299,300768625,33947283616,1634745,1716595,67896,203017437,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201118.csv b/output/alternative/sec/13f/universe/20201118.csv new file mode 100644 index 0000000..b5de105 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201118.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3303,9703658181,1119672532259,325797091,312145433,649898,6179411006,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,638,765566029,21049454289,10563968,18398838,0,493826192,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1110,418906161,18439585369,175912396,282681143,961076,312649177,32950852,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2559,215750239,316422676923,9583404,5812989,1434,140798562,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2781,236061761,333536431865,7831689,7593118,11926,157739890,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1912,496866517,60246711738,10596073,12966776,46773,301619567,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1489,286059255,42635199932,142405490,306027979,155421,183747359,29591001,1 +NB R735QTJ8XC9X,BAC,1,20200930,2012,5882996830,141296265242,124841975,157712018,1388986,4058622407,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1692,228482161,63312129395,178814774,312438469,104443,104375799,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2426,564913916,196939092921,391247330,567820720,543662,365825915,41754254,1 +USO THORT68ZZSYT,USO,1,20200930,184,36119919,1020101410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1308,301787509,34062555616,1703745,1793995,67896,204025069,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201119.csv b/output/alternative/sec/13f/universe/20201119.csv new file mode 100644 index 0000000..e23f866 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201119.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3314,9704601839,1119781812259,325797091,312145433,649898,6180003180,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,638,765566029,21049454289,10563968,18398838,0,493826192,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1116,419273124,18455765369,175912396,282685743,961076,312888545,32950852,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2565,215765803,316445550923,9583404,5812989,1434,140810632,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2789,236075135,333556031865,7831689,7593118,11926,157750452,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1919,496908029,60251763738,10596073,12966776,46773,301649518,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1493,286615446,42718510932,142418590,306030279,155421,184285427,29591001,1 +NB R735QTJ8XC9X,BAC,1,20200930,2016,5883093962,141298605242,124841975,157712018,1388986,4058683796,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1695,228488734,63313955395,178814774,312438969,104443,104379584,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2432,565090921,196998376921,391913330,568098620,543662,365927152,41754254,1 +USO THORT68ZZSYT,USO,1,20200930,184,36119919,1020101410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1309,301793757,34063262616,1703745,1793995,67896,204025069,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201120.csv b/output/alternative/sec/13f/universe/20201120.csv new file mode 100644 index 0000000..ceab727 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201120.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3326,9706041912,1119948586927,325799391,312302433,649898,6180139875,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,643,765648880,21051736065,10563968,18398838,0,493827192,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1119,419361170,18459646971,175912396,282685743,961076,312888545,32950852,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2574,215776426,316461147561,9583404,5812989,1434,140811875,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2799,236098098,333589686815,7831689,7609218,11926,157752655,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1925,496944801,60256237278,10596173,12966876,46773,301653878,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1499,286696224,42730610558,142418590,306030479,155421,184285427,29591001,1 +NB R735QTJ8XC9X,BAC,1,20200930,2024,5883587298,141310490312,124842175,157712018,1388986,4058720890,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1700,228733797,63382044415,178815074,312522269,104443,104381874,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2442,565479991,197128699599,391915330,568101920,543662,366127201,41754254,1 +USO THORT68ZZSYT,USO,1,20200930,184,36119919,1020101410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1318,301866502,34071495914,1703745,1793995,67896,204070459,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201123.csv b/output/alternative/sec/13f/universe/20201123.csv new file mode 100644 index 0000000..0c7b857 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201123.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3327,9706082057,1119952956927,325799391,312302433,649898,6180139875,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,643,765648880,21051736065,10563968,18398838,0,493827192,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1119,419361170,18459646971,175912396,282685743,961076,312888545,32950852,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2574,215776426,316461147561,9583404,5812989,1434,140811875,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2800,236100494,333593558815,7831689,7609218,11926,157752655,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1926,496944829,60256240278,10596173,12966876,46773,301653878,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1499,286696224,42730610558,142418590,306030479,155421,184285427,29591001,1 +NB R735QTJ8XC9X,BAC,1,20200930,2025,5883591082,141310580312,124842175,157712018,1388986,4058720890,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1701,228733902,63382072415,178815074,312522269,104443,104381874,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2443,565480119,197128741599,391915330,568101920,543662,366127201,41754254,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1318,301866502,34071495914,1703745,1793995,67896,204070459,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201124.csv b/output/alternative/sec/13f/universe/20201124.csv new file mode 100644 index 0000000..f4e5d6a --- /dev/null +++ b/output/alternative/sec/13f/universe/20201124.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3328,9706153389,1119961216927,325799391,312302433,649898,6180139875,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,643,765648880,21051736065,10563968,18398838,0,493827192,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1121,419395060,18461140971,175912396,282685743,961076,312888545,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2575,215776732,316461596561,9583404,5812989,1434,140811875,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2801,236102765,333596886815,7831689,7609218,11926,157752655,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1927,496947686,60256586278,10596173,12966876,46773,301653878,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1499,286696224,42730610558,142418590,306030479,155421,184285427,29591001,1 +NB R735QTJ8XC9X,BAC,1,20200930,2025,5883591082,141310580312,124842175,157712018,1388986,4058720890,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1701,228733902,63382072415,178815074,312522269,104443,104381874,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2444,565481974,197129362599,391915330,568101920,543662,366127201,41754254,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1319,301869588,34071844914,1703745,1793995,67896,204070459,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201125.csv b/output/alternative/sec/13f/universe/20201125.csv new file mode 100644 index 0000000..8542d1e --- /dev/null +++ b/output/alternative/sec/13f/universe/20201125.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3329,9706218273,1119968730927,325799391,312302433,649898,6180202939,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,643,765648880,21051736065,10563968,18398838,0,493827192,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1122,419402240,18461457971,175912396,282685743,961076,312895725,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2576,215777254,316462363561,9583404,5812989,1434,140812382,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2802,236104360,333599224815,7831689,7609218,11926,157754201,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1928,496951885,60257097278,10596173,12966876,46773,301658077,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1499,286696224,42730610558,142418590,306030479,155421,184285427,29591001,1 +NB R735QTJ8XC9X,BAC,1,20200930,2026,5883607072,141310965312,124842175,157712018,1388986,4058736580,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1701,228733902,63382072415,178815074,312522269,104443,104381874,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2445,565485875,197130668599,391915330,568101920,543662,366130858,41754254,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1319,301869588,34071844914,1703745,1793995,67896,204070459,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201126.csv b/output/alternative/sec/13f/universe/20201126.csv new file mode 100644 index 0000000..9cb3691 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201126.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3331,9707264545,1120089899927,325799391,312302433,649898,6181202491,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,643,765648880,21051736065,10563968,18398838,0,493827192,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1125,419827043,18480186971,175912396,282685743,961076,313169437,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2578,215790908,316482429561,9583404,5812989,1434,140825792,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2804,236122427,333625703815,7831689,7609218,11926,157771985,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1929,496959656,60258042278,10596173,12966876,46773,301658077,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1502,286727605,42735311558,142418590,306030479,155421,184295188,29591001,1 +NB R735QTJ8XC9X,BAC,1,20200930,2028,5884109393,141323066312,124842175,157712018,1388986,4059209734,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1703,228743065,63384618415,178815074,312522269,104443,104383506,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2448,565505691,197137305599,391915330,568101920,543662,366141374,41754254,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1320,301878152,34072813914,1703745,1793995,67896,204070459,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201127.csv b/output/alternative/sec/13f/universe/20201127.csv new file mode 100644 index 0000000..9cb3691 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201127.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3331,9707264545,1120089899927,325799391,312302433,649898,6181202491,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,643,765648880,21051736065,10563968,18398838,0,493827192,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1125,419827043,18480186971,175912396,282685743,961076,313169437,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2578,215790908,316482429561,9583404,5812989,1434,140825792,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2804,236122427,333625703815,7831689,7609218,11926,157771985,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1929,496959656,60258042278,10596173,12966876,46773,301658077,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1502,286727605,42735311558,142418590,306030479,155421,184295188,29591001,1 +NB R735QTJ8XC9X,BAC,1,20200930,2028,5884109393,141323066312,124842175,157712018,1388986,4059209734,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1703,228743065,63384618415,178815074,312522269,104443,104383506,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2448,565505691,197137305599,391915330,568101920,543662,366141374,41754254,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1320,301878152,34072813914,1703745,1793995,67896,204070459,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201130.csv b/output/alternative/sec/13f/universe/20201130.csv new file mode 100644 index 0000000..74ef47e --- /dev/null +++ b/output/alternative/sec/13f/universe/20201130.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3333,9707509622,1120118286927,325799391,312302433,649898,6181202491,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,643,765648880,21051736065,10563968,18398838,0,493827192,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1126,419889067,18482921971,175912396,282685743,961076,313169437,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2580,215798936,316494114561,9583404,5812989,1434,140825792,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2805,236122832,333626297815,7831689,7609218,11926,157771985,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1930,496964184,60258593278,10596173,12966876,46773,301658077,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1503,286754447,42739332558,142418590,306030479,155421,184295188,29591001,1 +NB R735QTJ8XC9X,BAC,1,20200930,2028,5884109393,141323066312,124842175,157712018,1388986,4059209734,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1706,228811099,63403701415,178815074,312522269,104443,104384875,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2449,565595759,197161598599,391915330,568101920,543662,366141374,41754254,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1320,301878152,34072813914,1703745,1793995,67896,204070459,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201201.csv b/output/alternative/sec/13f/universe/20201201.csv new file mode 100644 index 0000000..938c4e3 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201201.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3338,9708708022,1120257221927,325799391,312302433,649898,6181711307,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,644,765649431,21051758065,10563968,18398838,0,493827743,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1128,419915680,18484105971,175912396,282685743,961076,313171187,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2583,215801107,316497337561,9583404,5812989,1434,140826116,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2808,236124187,333628351815,7831689,7609218,11926,157772196,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1933,496988716,60261622278,10596173,12966876,46773,301674708,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1506,287324631,42824741558,142418590,306030479,155421,184832981,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2033,5885022783,141345077312,124842175,157712018,1388986,4059971644,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1709,229111347,63487184415,178815074,312522269,104443,104487213,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2454,565972279,197287799599,391915330,568101920,543662,366352055,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1321,301886179,34073721914,1703745,1793995,67896,204070459,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201202.csv b/output/alternative/sec/13f/universe/20201202.csv new file mode 100644 index 0000000..12dc7a5 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201202.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3341,9709977689,1120404255927,325799391,312302433,649898,6182945414,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,644,765649431,21051758065,10563968,18398838,0,493827743,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1129,419920918,18484336971,175912396,282685743,961076,313174990,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2585,215813063,316514896561,9583404,5812989,1434,140837806,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2810,236172416,333699044815,7831689,7609218,11926,157819357,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1934,496997327,60262670278,10596173,12966876,46773,301683219,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1507,287333222,42826028558,142418590,306030479,155421,184836877,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2034,5885045572,141346080312,124842175,157712018,1388986,4059993409,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1711,229118427,63489146415,178815074,312522269,104443,104494077,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2456,566076852,197322822599,391915330,568101920,543662,366418141,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1322,301888476,34073981914,1703745,1793995,67896,204072756,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201203.csv b/output/alternative/sec/13f/universe/20201203.csv new file mode 100644 index 0000000..08b2579 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201203.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3342,9712809937,1120732258927,325799391,312302433,649898,6185777662,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,645,765726206,21053872065,10563968,18398838,0,493904518,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1129,419920918,18484336971,175912396,282685743,961076,313174990,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2586,215870995,316600033561,9583404,5812989,1434,140895738,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2811,236261902,333830195815,7831689,7609218,11926,157908843,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1935,497076908,60272353278,10596173,12966876,46773,301762800,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1507,287333222,42826028558,142418590,306030479,155421,184836877,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2035,5887138735,141396504312,124842175,157712018,1388986,4062086572,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1711,229118427,63489146415,178815074,312522269,104443,104494077,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2457,566155427,197349136599,391915330,568101920,543662,366496716,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1323,301923381,34077931914,1703745,1793995,67896,204107661,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201204.csv b/output/alternative/sec/13f/universe/20201204.csv new file mode 100644 index 0000000..b098dc0 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201204.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3346,9713872709,1120855337927,325799391,312302433,649898,6186418757,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,646,765885617,21058261065,10563968,18398838,0,494063929,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1131,419967138,18486374971,175912396,282685743,961076,313221172,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2590,215899437,316641831561,9583404,5812989,1434,140913262,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2815,236279727,333856318815,7831689,7609218,11926,157921919,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1939,497153527,60281674278,10596173,12966876,46773,301785500,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1508,287370068,42831547558,142418590,306030479,155421,184873723,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2038,5888860464,141437980312,124842175,157712018,1388986,4063684097,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1714,229219625,63517263415,178815074,312522269,104443,104501069,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2461,566546411,197480073599,391915330,568101920,543662,366662160,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1324,301959527,34082022914,1703745,1793995,67896,204143807,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201207.csv b/output/alternative/sec/13f/universe/20201207.csv new file mode 100644 index 0000000..b098dc0 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201207.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3346,9713872709,1120855337927,325799391,312302433,649898,6186418757,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,646,765885617,21058261065,10563968,18398838,0,494063929,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1131,419967138,18486374971,175912396,282685743,961076,313221172,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2590,215899437,316641831561,9583404,5812989,1434,140913262,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2815,236279727,333856318815,7831689,7609218,11926,157921919,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1939,497153527,60281674278,10596173,12966876,46773,301785500,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1508,287370068,42831547558,142418590,306030479,155421,184873723,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2038,5888860464,141437980312,124842175,157712018,1388986,4063684097,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1714,229219625,63517263415,178815074,312522269,104443,104501069,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2461,566546411,197480073599,391915330,568101920,543662,366662160,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1324,301959527,34082022914,1703745,1793995,67896,204143807,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201208.csv b/output/alternative/sec/13f/universe/20201208.csv new file mode 100644 index 0000000..cbe3639 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201208.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3347,9713920533,1120860925927,325799391,312302433,649898,6186422616,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,646,765885617,21058261065,10563968,18398838,0,494063929,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1131,419967138,18486374971,175912396,282685743,961076,313221172,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2591,215900213,316643067561,9583404,5812989,1434,140913293,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2816,236280041,333856816815,7831689,7609218,11926,157921942,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1939,497153527,60281674278,10596173,12966876,46773,301785500,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1508,287370068,42831547558,142418590,306030479,155421,184873723,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2038,5888860464,141437980312,124842175,157712018,1388986,4063684097,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1715,229221042,63517666415,178815074,312522269,104443,104501152,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2462,566548934,197480939599,391915330,568101920,543662,366662744,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1325,301967753,34082961914,1703745,1793995,67896,204144250,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201209.csv b/output/alternative/sec/13f/universe/20201209.csv new file mode 100644 index 0000000..c9e6482 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201209.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3348,9713973161,1120867020927,325799391,312302433,649898,6186475244,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,647,766231601,21067786065,10563968,18398838,0,494409913,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1131,419967138,18486374971,175912396,282685743,961076,313221172,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2592,215907806,316654226561,9583404,5812989,1434,140920886,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2817,236283166,333861396815,7831689,7609218,11926,157925067,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1940,497161383,60282630278,10596173,12966876,46773,301793356,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1508,287370068,42831547558,142418590,306030479,155421,184873723,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2039,5889159843,141445192312,124842175,157712018,1388986,4063983476,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1715,229221042,63517666415,178815074,312522269,104443,104501152,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2462,566548934,197480939599,391915330,568101920,543662,366662744,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1326,301970602,34083283914,1703745,1793995,67896,204147099,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201210.csv b/output/alternative/sec/13f/universe/20201210.csv new file mode 100644 index 0000000..b0713ea --- /dev/null +++ b/output/alternative/sec/13f/universe/20201210.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3349,9713995282,1120869582927,325799391,312302433,649898,6186475244,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,647,766231601,21067786065,10563968,18398838,0,494409913,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1131,419967138,18486374971,175912396,282685743,961076,313221172,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2592,215907806,316654226561,9583404,5812989,1434,140920886,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2817,236283166,333861396815,7831689,7609218,11926,157925067,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1940,497161383,60282630278,10596173,12966876,46773,301793356,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1509,287377659,42832684558,142418590,306030479,155421,184873723,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2039,5889159843,141445192312,124842175,157712018,1388986,4063983476,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1716,229222135,63517970415,178815074,312522269,104443,104501152,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2462,566548934,197480939599,391915330,568101920,543662,366662744,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1327,301972674,34083517914,1703745,1793995,67896,204147099,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201211.csv b/output/alternative/sec/13f/universe/20201211.csv new file mode 100644 index 0000000..1dd1205 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201211.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3352,9715834519,1121086209927,325799391,312302433,649898,6188314481,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,649,766240363,21068027065,10563968,18398838,0,494418675,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1133,419994183,18487566971,175912396,282685743,961076,313248217,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2594,215916894,316667581561,9583404,5812989,1434,140929974,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2819,236294670,333878247815,7831689,7609218,11926,157936571,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1942,497221340,60289925278,10596173,12966876,46773,301853313,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1510,287420976,42839172558,142418590,306030479,155421,184917040,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2041,5889628903,141456492312,124842175,157712018,1388986,4064452536,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1718,229303236,63540503415,178815074,312522269,104443,104582253,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2464,566727744,197540820599,391915330,568101920,543662,366841554,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1329,301997996,34086382914,1703745,1793995,67896,204172421,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201214.csv b/output/alternative/sec/13f/universe/20201214.csv new file mode 100644 index 0000000..d4f0603 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201214.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3355,9716666988,1121182617927,325799391,312302433,649898,6188987784,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,649,766240363,21068027065,10563968,18398838,0,494418675,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1135,420027588,18489040971,175912396,282685743,961076,313281583,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2597,215928637,316684839561,9583404,5812989,1434,140939167,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2823,236329682,333929561815,7831689,7609218,11926,157970286,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1945,497270879,60295952278,10596173,12966876,46773,301871002,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1511,287424872,42839756558,142418590,306030479,155421,184920936,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2043,5889779934,141460130312,124842175,157712018,1388986,4064603567,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1718,229303236,63540503415,178815074,312522269,104443,104582253,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2465,566736187,197543648599,391915330,568101920,543662,366849547,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1331,302109583,34099011914,1703745,1793995,67896,204282839,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201215.csv b/output/alternative/sec/13f/universe/20201215.csv new file mode 100644 index 0000000..3cdb4fa --- /dev/null +++ b/output/alternative/sec/13f/universe/20201215.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3358,9727385023,1122423872927,325799391,312302433,649898,6198837491,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,650,766580554,21077392065,10563968,18398838,0,494758866,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1137,420825306,18524211971,175912396,282685743,961076,314061163,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2599,216126116,316975054561,9583404,5812989,1434,141135478,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2826,236540485,334238514815,7831689,7609218,11926,158162819,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1948,497735599,60352494278,10596173,12966876,46773,302287242,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1512,287429573,42840460558,142418590,306030479,155421,184924196,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2045,5894249451,141567801312,124842175,157712018,1388986,4069024287,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1721,229310785,63542600415,178815074,312562269,104443,104584030,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2469,566741967,197545582599,391925330,568142820,543662,366851599,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1333,302286848,34119072914,1703745,1793995,67896,204456989,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201216.csv b/output/alternative/sec/13f/universe/20201216.csv new file mode 100644 index 0000000..8213693 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201216.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3359,9727425775,1122428592927,325799391,312302433,649898,6198878243,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,650,766580554,21077392065,10563968,18398838,0,494758866,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1137,420825306,18524211971,175912396,282685743,961076,314061163,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2600,216128833,316979047561,9583404,5812989,1434,141138195,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2827,236542910,334242068815,7831689,7609218,11926,158165244,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1948,497735599,60352494278,10596173,12966876,46773,302287242,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1512,287429573,42840460558,142418590,306030479,155421,184924196,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2045,5894249451,141567801312,124842175,157712018,1388986,4069024287,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1721,229310785,63542600415,178815074,312562269,104443,104584030,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2469,566741967,197545582599,391925330,568142820,543662,366851599,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1334,302302368,34120829914,1703745,1793995,67896,204472509,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201217.csv b/output/alternative/sec/13f/universe/20201217.csv new file mode 100644 index 0000000..8213693 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201217.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3359,9727425775,1122428592927,325799391,312302433,649898,6198878243,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,650,766580554,21077392065,10563968,18398838,0,494758866,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1137,420825306,18524211971,175912396,282685743,961076,314061163,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2600,216128833,316979047561,9583404,5812989,1434,141138195,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2827,236542910,334242068815,7831689,7609218,11926,158165244,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1948,497735599,60352494278,10596173,12966876,46773,302287242,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1512,287429573,42840460558,142418590,306030479,155421,184924196,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2045,5894249451,141567801312,124842175,157712018,1388986,4069024287,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1721,229310785,63542600415,178815074,312562269,104443,104584030,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2469,566741967,197545582599,391925330,568142820,543662,366851599,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1334,302302368,34120829914,1703745,1793995,67896,204472509,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201218.csv b/output/alternative/sec/13f/universe/20201218.csv new file mode 100644 index 0000000..8213693 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201218.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3359,9727425775,1122428592927,325799391,312302433,649898,6198878243,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,650,766580554,21077392065,10563968,18398838,0,494758866,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1137,420825306,18524211971,175912396,282685743,961076,314061163,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2600,216128833,316979047561,9583404,5812989,1434,141138195,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2827,236542910,334242068815,7831689,7609218,11926,158165244,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1948,497735599,60352494278,10596173,12966876,46773,302287242,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1512,287429573,42840460558,142418590,306030479,155421,184924196,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2045,5894249451,141567801312,124842175,157712018,1388986,4069024287,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1721,229310785,63542600415,178815074,312562269,104443,104584030,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2469,566741967,197545582599,391925330,568142820,543662,366851599,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1334,302302368,34120829914,1703745,1793995,67896,204472509,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201221.csv b/output/alternative/sec/13f/universe/20201221.csv new file mode 100644 index 0000000..a09b4b5 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201221.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3361,9728069924,1122505176927,325799391,312312833,649898,6198938718,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,650,766580554,21077392065,10563968,18398838,0,494758866,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1137,420825306,18524211971,175912396,282685743,961076,314061163,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2601,216136920,316993197561,9583404,5812989,1434,141138195,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2829,236548989,334251405815,7831689,7609218,11926,158169778,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1948,497735599,60352494278,10596173,12966876,46773,302287242,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1512,287429573,42840460558,142418590,306030479,155421,184924196,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2047,5894279272,141568565312,124842175,157712018,1388986,4069036381,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1722,229341896,63551573415,178815074,312562269,104443,104584030,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2470,566749043,197548082599,391937430,568142820,543662,366851599,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1335,302304557,34121094914,1703745,1793995,67896,204472509,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201222.csv b/output/alternative/sec/13f/universe/20201222.csv new file mode 100644 index 0000000..51e7183 --- /dev/null +++ b/output/alternative/sec/13f/universe/20201222.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3362,9729976277,1122725951927,325799391,312312833,649898,6200765013,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,651,766586617,21077559065,10563968,18398838,0,494764929,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1138,420876787,18526481971,175912396,282685743,961076,314112644,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2602,216185603,317064742561,9583404,5812989,1434,141186726,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2830,236576911,334292327815,7831689,7609218,11926,158193673,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1949,497895752,60371980278,10596173,12966876,46773,302447358,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1513,287454299,42844164558,142418590,306030479,155421,184948922,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2048,5894645823,141577395312,124842175,157712018,1388986,4069402932,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1723,229343256,63551951415,178815074,312562269,104443,104585390,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2471,566883706,197593179599,391937430,568142820,543662,366986262,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1336,302306665,34121333914,1703745,1793995,67896,204474617,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201223.csv b/output/alternative/sec/13f/universe/20201223.csv new file mode 100644 index 0000000..6e148fe --- /dev/null +++ b/output/alternative/sec/13f/universe/20201223.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3363,9729982640,1122726688927,325799391,312312833,649898,6200771376,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,651,766586617,21077559065,10563968,18398838,0,494764929,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1138,420876787,18526481971,175912396,282685743,961076,314112644,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2602,216185603,317064742561,9583404,5812989,1434,141186726,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2830,236576911,334292327815,7831689,7609218,11926,158193673,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1949,497895752,60371980278,10596173,12966876,46773,302447358,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1514,287457891,42844702558,142418590,306030479,155421,184952514,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2048,5894645823,141577395312,124842175,157712018,1388986,4069402932,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1724,229347747,63553199415,178815074,312562269,104443,104589881,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2471,566883706,197593179599,391937430,568142820,543662,366986262,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1336,302306665,34121333914,1703745,1793995,67896,204474617,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201224.csv b/output/alternative/sec/13f/universe/20201224.csv new file mode 100644 index 0000000..5f44d5b --- /dev/null +++ b/output/alternative/sec/13f/universe/20201224.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3365,9729994429,1122728053927,325799391,312312833,649898,6200771376,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,651,766586617,21077559065,10563968,18398838,0,494764929,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1138,420876787,18526481971,175912396,282685743,961076,314112644,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2602,216185603,317064742561,9583404,5812989,1434,141186726,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2830,236576911,334292327815,7831689,7609218,11926,158193673,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1949,497895752,60371980278,10596173,12966876,46773,302447358,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1514,287457891,42844702558,142418590,306030479,155421,184952514,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2048,5894645823,141577395312,124842175,157712018,1388986,4069402932,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1724,229347747,63553199415,178815074,312562269,104443,104589881,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2473,566887134,197594327599,391937430,568142820,543662,366986262,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1336,302306665,34121333914,1703745,1793995,67896,204474617,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201225.csv b/output/alternative/sec/13f/universe/20201225.csv new file mode 100644 index 0000000..5f44d5b --- /dev/null +++ b/output/alternative/sec/13f/universe/20201225.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3365,9729994429,1122728053927,325799391,312312833,649898,6200771376,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,651,766586617,21077559065,10563968,18398838,0,494764929,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1138,420876787,18526481971,175912396,282685743,961076,314112644,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2602,216185603,317064742561,9583404,5812989,1434,141186726,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2830,236576911,334292327815,7831689,7609218,11926,158193673,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1949,497895752,60371980278,10596173,12966876,46773,302447358,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1514,287457891,42844702558,142418590,306030479,155421,184952514,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2048,5894645823,141577395312,124842175,157712018,1388986,4069402932,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1724,229347747,63553199415,178815074,312562269,104443,104589881,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2473,566887134,197594327599,391937430,568142820,543662,366986262,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1336,302306665,34121333914,1703745,1793995,67896,204474617,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201228.csv b/output/alternative/sec/13f/universe/20201228.csv new file mode 100644 index 0000000..5f44d5b --- /dev/null +++ b/output/alternative/sec/13f/universe/20201228.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3365,9729994429,1122728053927,325799391,312312833,649898,6200771376,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,651,766586617,21077559065,10563968,18398838,0,494764929,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1138,420876787,18526481971,175912396,282685743,961076,314112644,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2602,216185603,317064742561,9583404,5812989,1434,141186726,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2830,236576911,334292327815,7831689,7609218,11926,158193673,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1949,497895752,60371980278,10596173,12966876,46773,302447358,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1514,287457891,42844702558,142418590,306030479,155421,184952514,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2048,5894645823,141577395312,124842175,157712018,1388986,4069402932,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1724,229347747,63553199415,178815074,312562269,104443,104589881,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2473,566887134,197594327599,391937430,568142820,543662,366986262,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1336,302306665,34121333914,1703745,1793995,67896,204474617,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201229.csv b/output/alternative/sec/13f/universe/20201229.csv new file mode 100644 index 0000000..5f44d5b --- /dev/null +++ b/output/alternative/sec/13f/universe/20201229.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3365,9729994429,1122728053927,325799391,312312833,649898,6200771376,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,651,766586617,21077559065,10563968,18398838,0,494764929,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1138,420876787,18526481971,175912396,282685743,961076,314112644,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2602,216185603,317064742561,9583404,5812989,1434,141186726,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2830,236576911,334292327815,7831689,7609218,11926,158193673,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1949,497895752,60371980278,10596173,12966876,46773,302447358,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1514,287457891,42844702558,142418590,306030479,155421,184952514,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2048,5894645823,141577395312,124842175,157712018,1388986,4069402932,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1724,229347747,63553199415,178815074,312562269,104443,104589881,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2473,566887134,197594327599,391937430,568142820,543662,366986262,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1336,302306665,34121333914,1703745,1793995,67896,204474617,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201230.csv b/output/alternative/sec/13f/universe/20201230.csv new file mode 100644 index 0000000..1b285ea --- /dev/null +++ b/output/alternative/sec/13f/universe/20201230.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3366,9730059060,1122735537927,325799391,312312833,649898,6200836007,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,651,766586617,21077559065,10563968,18398838,0,494764929,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1139,420970464,18530611971,175912396,282685743,961076,314206321,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2603,216185801,317065032561,9583404,5812989,1434,141186924,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2831,236577253,334292828815,7831689,7609218,11926,158194015,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1949,497895752,60371980278,10596173,12966876,46773,302447358,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1515,287496788,42850528558,142418590,306030479,155421,184991411,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2049,5894666698,141577897312,124842175,157712018,1388986,4069423807,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1725,229397349,63566980415,178815074,312562269,104443,104639483,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2474,566953969,197616708599,391937430,568142820,543662,367053097,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1336,302306665,34121333914,1703745,1793995,67896,204474617,18262914,1 diff --git a/output/alternative/sec/13f/universe/20201231.csv b/output/alternative/sec/13f/universe/20201231.csv new file mode 100644 index 0000000..1b285ea --- /dev/null +++ b/output/alternative/sec/13f/universe/20201231.csv @@ -0,0 +1,13 @@ +AAPL R735QTJ8XC9X,AAPL,1,20200930,3366,9730059060,1122735537927,325799391,312312833,649898,6200836007,641098575,1 +AIG R735QTJ8XC9X,AIG,1,20200930,651,766586617,21077559065,10563968,18398838,0,494764929,72508312,1 +BNO UN3IMQ2JU1YD,BNO,1,20200930,39,8288042,87717888,230500,232700,0,4546672,8048,0 +EEM SNQLASP67O85,EEM,1,20200930,1139,420970464,18530611971,175912396,282685743,961076,314206321,32950877,1 +GOOCV VP83T1ZUHROL,GOOG,1,20200930,2603,216185801,317065032561,9583404,5812989,1434,141186924,12191671,1 +GOOG T1AZ164W5VTX,GOOGL,1,20200930,2831,236577253,334292828815,7831689,7609218,11926,158194015,18783016,1 +IBM R735QTJ8XC9X,IBM,1,20200930,1949,497895752,60371980278,10596173,12966876,46773,302447358,40758566,1 +IWM RV0PWMLXVHPH,IWM,1,20200930,1515,287496788,42850528558,142418590,306030479,155421,184991411,29591795,1 +NB R735QTJ8XC9X,BAC,1,20200930,2049,5894666698,141577897312,124842175,157712018,1388986,4069423807,485120593,1 +QQQ RIWIV7K5Z9LX,QQQ,1,20200930,1725,229397349,63566980415,178815074,312562269,104443,104639483,32345398,1 +SPY R735QTJ8XC9X,SPY,1,20200930,2474,566953969,197616708599,391937430,568142820,543662,367053097,41754440,1 +USO THORT68ZZSYT,USO,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 +UW R735QTJ8XC9X,WM,1,20200930,1336,302306665,34121333914,1703745,1793995,67896,204474617,18262914,1 diff --git a/output/alternative/sec/13f/uso.csv b/output/alternative/sec/13f/uso.csv new file mode 100644 index 0000000..d212d0e --- /dev/null +++ b/output/alternative/sec/13f/uso.csv @@ -0,0 +1,31 @@ +20201006 03:00,1,20200930,1,37,1000,0,0,0,0,0,0 +20201007 03:00,1,20200930,5,4406,124000,0,0,0,773,0,0 +20201008 03:00,1,20200930,6,4435,125000,0,0,0,802,0,0 +20201009 03:00,1,20200930,8,11815,334000,0,0,0,7936,0,0 +20201010 03:00,1,20200930,11,36427,1030000,450,0,0,28081,0,0 +20201014 03:00,1,20200930,17,83368,2357000,450,0,0,49643,0,0 +20201015 03:00,1,20200930,21,439203,12423000,450,0,0,404754,0,0 +20201016 03:00,1,20200930,24,464083,13128000,450,0,0,405804,0,0 +20201017 03:00,1,20200930,25,464681,13145000,450,0,0,405804,0,0 +20201020 03:00,1,20200930,29,472805,13374000,32250,120,0,413832,0,0 +20201021 03:00,1,20200930,32,473590,13396000,32250,120,0,413832,0,0 +20201022 03:00,1,20200930,35,482492,13648000,32250,120,0,421580,0,0 +20201023 03:00,1,20200930,38,510085,14429000,32250,120,0,437866,0,0 +20201024 03:00,1,20200930,39,510710,14447000,32250,120,0,438491,0,0 +20201027 03:00,1,20200930,44,757021,21415000,36750,120,0,684658,0,0 +20201028 03:00,1,20200930,49,856032,24215000,2632750,1434420,0,780031,0,0 +20201029 03:00,1,20200930,52,856911,24240000,2689750,1434420,0,780642,0,0 +20201030 03:00,1,20200930,55,980643,27741000,2689750,1434420,0,806626,0,0 +20201031 03:00,1,20200930,59,987093,27923000,2689750,1434420,416,806626,0,0 +20201103 03:00,1,20200930,62,1005774,28451000,2689750,1434420,416,807384,0,0 +20201104 03:00,1,20200930,68,1056817,29896000,2709750,1434420,416,810592,0,1 +20201105 03:00,1,20200930,72,2287581,64271000,2742150,1434420,416,844922,0,1 +20201106 03:00,1,20200930,78,2532546,71200000,2742150,1434420,416,990815,0,1 +20201107 03:00,1,20200930,87,2573265,72351000,2742650,1434420,416,1001213,0,1 +20201110 03:00,1,20200930,93,3372023,94949000,2751150,1434420,416,1397449,0,1 +20201111 03:00,1,20200930,101,3421906,96358590,2751150,1434420,416,1408231,0,1 +20201113 03:00,1,20200930,130,6223988,174622278,4662310,1941548,416,3482183,584302,1 +20201114 03:00,1,20200930,158,12551111,353323410,8822434,8868420,416,8889941,717430,1 +20201117 03:00,1,20200930,179,33782626,953980410,22853426,38616768,69466,30100605,717495,1 +20201118 03:00,1,20200930,184,36119919,1020101410,23671038,39827196,69466,32436364,717495,1 +20201121 03:00,2,20200331,229,153382774,648188610,50021965,94412340,12640,140040845,3985466,1,20200930,186,36121055,1020133410,23671038,39827196,69466,32436364,717495,1 diff --git a/output/alternative/sec/13f/wm.csv b/output/alternative/sec/13f/wm.csv new file mode 100644 index 0000000..1498297 --- /dev/null +++ b/output/alternative/sec/13f/wm.csv @@ -0,0 +1,50 @@ +20201002 03:00,1,20200930,5,162449,18383000,0,0,0,150636,0,0 +20201003 03:00,1,20200930,6,180919,20474000,0,0,0,169069,0,0 +20201006 03:00,1,20200930,13,368332,41682000,0,0,0,308597,1385,0 +20201007 03:00,1,20200930,33,609798,69007000,0,0,0,361654,1385,0 +20201008 03:00,3,20200331,1304,310795327,28452799685,1562445,1442663,67183,207204739,20791506,1,20200630,1324,304453563,32195233671,1541045,1920563,53618,198444841,19384570,1,20200930,40,669387,75751000,0,0,0,416004,4354,0 +20201009 03:00,1,20200930,59,1111072,125738071,0,0,0,753362,25118,0 +20201010 03:00,1,20200930,75,1825671,206609071,0,0,0,1355389,30268,0 +20201014 03:00,1,20200930,117,2856384,323540071,0,0,0,2086826,30268,1 +20201015 03:00,1,20200930,144,3639389,412152311,0,0,53096,2692685,129248,1 +20201016 03:00,1,20200930,162,4734636,536191311,0,0,53096,3647684,131227,1 +20201017 03:00,1,20200930,183,5185078,585922311,0,1000,53096,3939434,137785,1 +20201020 03:00,1,20200930,211,6517398,736693311,0,1000,53096,5019001,139722,1 +20201021 03:00,1,20200930,238,8087201,914346311,0,36044,53096,6424636,139722,1 +20201022 03:00,1,20200930,266,9075043,1026143311,0,36044,59780,7313293,142722,1 +20201023 03:00,1,20200930,283,9485921,1072658311,0,36044,59780,7526650,142722,1 +20201024 03:00,1,20200930,303,10903524,1233085311,0,37857,59780,8320874,142722,1 +20201027 03:00,1,20200930,336,11717079,1325164921,0,39057,59906,9076919,142722,1 +20201028 03:00,1,20200930,372,12342844,1395887921,25100,50657,59906,9503136,142897,1 +20201029 03:00,1,20200930,404,13215888,1494709921,25100,50657,59906,9912443,425065,1 +20201030 03:00,1,20200930,435,15921640,1801223921,25100,50657,59906,11320844,434224,1 +20201031 03:00,1,20200930,475,19388951,2194015344,25145,51495,59906,14419184,630177,1 +20201103 03:00,1,20200930,506,20535980,2323788344,25145,51495,59906,15043659,632064,1 +20201104 03:00,1,20200930,554,22453888,2540670928,25145,51495,59906,16363793,766253,1 +20201105 03:00,1,20200930,587,25288258,2861474928,25145,51495,59906,18243958,831910,1 +20201106 03:00,1,20200930,625,38638823,4372188928,25145,51495,59906,30447487,857380,1 +20201107 03:00,2,20200630,1325,304456491,32195543671,1541045,1920563,53618,198447769,19384570,1,20200930,668,74783369,8410098241,25145,51495,59906,61856436,1450480,1 +20201110 03:00,1,20200930,728,85129921,9581386241,25145,51495,59906,67534975,1797595,1 +20201111 03:00,1,20200930,790,108785914,12258610204,25145,51495,60206,86200372,4190674,1 +20201113 03:00,1,20200930,986,154286920,17376098914,128745,277795,60206,120266607,5326730,1 +20201114 03:00,2,20190930,1291,313141015,35937501114,1232061,1911527,54496,223984828,20486829,1,20200930,1164,216353749,24393874616,367845,542295,60206,166423211,8794925,1 +20201117 03:00,2,20200630,1326,304483231,32198375671,1541045,1920563,53618,198447769,19384570,1,20200930,1299,300768625,33947283616,1634745,1716595,67896,203017437,18262914,1 +20201118 03:00,1,20200930,1308,301787509,34062555616,1703745,1793995,67896,204025069,18262914,1 +20201119 03:00,2,20200630,1327,304485531,32198889671,1541045,1920563,53618,198450069,19384570,1,20200930,1309,301793757,34063262616,1703745,1793995,67896,204025069,18262914,1 +20201120 03:00,1,20200930,1318,301866502,34071495914,1703745,1793995,67896,204070459,18262914,1 +20201121 03:00,1,20200630,1328,304487766,32199126475,1541045,1920563,53618,198450069,19384570,1 +20201124 03:00,1,20200930,1319,301869588,34071844914,1703745,1793995,67896,204070459,18262914,1 +20201126 03:00,1,20200930,1320,301878152,34072813914,1703745,1793995,67896,204070459,18262914,1 +20201201 03:00,1,20200930,1321,301886179,34073721914,1703745,1793995,67896,204070459,18262914,1 +20201202 03:00,1,20200930,1322,301888476,34073981914,1703745,1793995,67896,204072756,18262914,1 +20201203 03:00,1,20200930,1323,301923381,34077931914,1703745,1793995,67896,204107661,18262914,1 +20201204 03:00,1,20200930,1324,301959527,34082022914,1703745,1793995,67896,204143807,18262914,1 +20201208 03:00,1,20200930,1325,301967753,34082961914,1703745,1793995,67896,204144250,18262914,1 +20201209 03:00,1,20200930,1326,301970602,34083283914,1703745,1793995,67896,204147099,18262914,1 +20201210 03:00,1,20200930,1327,301972674,34083517914,1703745,1793995,67896,204147099,18262914,1 +20201211 03:00,1,20200930,1329,301997996,34086382914,1703745,1793995,67896,204172421,18262914,1 +20201212 03:00,1,20200930,1331,302109583,34099011914,1703745,1793995,67896,204282839,18262914,1 +20201215 03:00,1,20200930,1333,302286848,34119072914,1703745,1793995,67896,204456989,18262914,1 +20201216 03:00,2,20200630,1329,304503558,32200798475,1541045,1920563,53618,198465861,19384570,1,20200930,1334,302302368,34120829914,1703745,1793995,67896,204472509,18262914,1 +20201219 03:00,1,20200930,1335,302304557,34121094914,1703745,1793995,67896,204472509,18262914,1 +20201222 03:00,1,20200930,1336,302306665,34121333914,1703745,1793995,67896,204474617,18262914,1 diff --git a/tests/ProtobufSubTypes.cs b/tests/ProtobufSubTypes.cs new file mode 100644 index 0000000..dcf1da6 --- /dev/null +++ b/tests/ProtobufSubTypes.cs @@ -0,0 +1,43 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; +using ProtoBuf.Meta; +using QuantConnect.Data; +using QuantConnect.DataSource; + +namespace QuantConnect.DataLibrary.Tests +{ + /// + /// Registers this repository's data types with protobuf, which is what LEAN's live data + /// distributor does before it serializes a point. + /// + /// It happens here, once, rather than in each round trip test: the model freezes the moment a + /// serializer is generated for , so the first test to serialize anything + /// would leave every later registration throwing. The field numbers are this repository's to + /// choose and have to stay clear of each other and of the ones BaseData gives its own + /// subclasses, which reach 400. + /// + [SetUpFixture] + public class ProtobufSubTypes + { + [OneTimeSetUp] + public void RegisterSubTypes() + { + RuntimeTypeModel.Default[typeof(BaseData)].AddSubType(2000, typeof(SECReport8K)); + RuntimeTypeModel.Default[typeof(BaseData)].AddSubType(2001, typeof(SEC13F)); + } + } +} diff --git a/tests/SEC13FProcessorTests.cs b/tests/SEC13FProcessorTests.cs new file mode 100644 index 0000000..239dae0 --- /dev/null +++ b/tests/SEC13FProcessorTests.cs @@ -0,0 +1,1954 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using NUnit.Framework; +using QuantConnect.Configuration; +using QuantConnect.DataProcessing; +using QuantConnect.DataSource; +using QuantConnect.Securities; + +namespace QuantConnect.DataLibrary.Tests +{ + /// + /// Unit tests for the 13F processor. + /// + /// The data classes had full coverage while the processor, which is where the subtle logic + /// lives, had none. Every case below is a defect that actually reached a run and was caught by + /// hand: a CUSIP left-padded into a different security, a thousand-fold step in the middle of + /// the value history, a file written in quarter order that LEAN then silently truncated, and a + /// universe that either threw away the finished quarter or resurrected a retired one. + /// + [TestFixture] + public class SEC13FProcessorTests + { + /// The filing date the counting tests file on, when the date itself is not the point. + private static readonly DateTime Filed = new(2024, 2, 14); + + private string _root; + private bool _configSeeded; + private string _previousDataFolder; + private string _previousLookupDate; + + [SetUp] + public void SetUp() + { + _root = Path.Combine(Path.GetTempPath(), $"sec-13f-tests-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_root); + } + + [TearDown] + public void TearDown() + { + if (_configSeeded) + { + // The data folder is process wide and the resolver caches the one it was built with, + // so a fixture left behind would be read by whatever runs next. A key that was absent + // goes back to absent: written back as an empty string it left Globals.DataFolder + // empty for every later test in the process. + Config.Reset(); + if (_previousDataFolder != null) + { + Config.Set("data-folder", _previousDataFolder); + } + + if (_previousLookupDate != null) + { + Config.Set("map-file-provider-lookup-date", _previousLookupDate); + } + + Globals.Reset(); + _configSeeded = false; + } + + if (Directory.Exists(_root)) + { + Directory.Delete(_root, true); + } + } + + // ---- CUSIP normalisation -------------------------------------------------------------- + + [TestCase("037833100")] // Apple + [TestCase("464287226")] // iShares Core US Aggregate Bond + [TestCase("72201R205")] + public void ANineCharacterCusipIsLeftAlone(string cusip) + { + // Most reported CUSIPs already carry their check digit and must not be touched at all. + Assert.AreEqual(cusip, SEC13FDownloader.NormalizeCusip(cusip)); + } + + [TestCase("37833100", "037833100")] // Apple, one leading zero eaten + [TestCase("2824100", "002824100")] // Abbott, two eaten + [TestCase("84670702", "084670702")] + public void AShortCusipMissingItsLeadingZerosIsPadded(string reported, string expected) + { + // The full history carries 12,372 short CUSIPs, most of them an identifier a spreadsheet + // somewhere read as a number and stripped the leading zeros off. + Assert.AreEqual(expected, SEC13FDownloader.NormalizeCusip(reported)); + } + + [TestCase("46428722", "464287226")] // iShares Core US Aggregate Bond + [TestCase("72201R20", "72201R205")] + public void AShortCusipMissingItsCheckDigitIsCompletedInstead(string reported, string expected) + { + // The other reason a CUSIP arrives short, and it needs the opposite repair. Left-padding + // "46428722" would have produced 046428722, a perfectly well-formed identifier belonging + // to a different security, so the two cases are told apart by the check digit itself: + // the padded reading is only accepted when it checks out. + Assert.AreEqual(expected, SEC13FDownloader.NormalizeCusip(reported)); + } + + [TestCase("037833100")] // Apple + [TestCase("594918104")] // Microsoft + [TestCase("67066G104")] // NVIDIA, the letter case + [TestCase("464287226")] // iShares Core US Aggregate Bond + [TestCase("002824100")] // Abbott, the leading-zero case + public void TheCheckDigitAgreesWithRealCusips(string cusip) + { + // Everything above rests on the check digit being right, so it is held against real + // published identifiers rather than against itself. + Assert.AreEqual(cusip[8], SEC13FDownloader.ComputeCusipCheckDigit(cusip.Substring(0, 8)), + $"the computed check digit disagrees with the published {cusip}"); + } + + [TestCase("")] + [TestCase("0")] + [TestCase("ABC")] + [TestCase("12345")] + public void JunkTooShortToBeACusipIsRejected(string cusip) + { + // Below six characters there is not enough of an identifier left to repair, and guessing + // one would attribute somebody's positions to whichever security the guess landed on. + Assert.IsNull(SEC13FDownloader.NormalizeCusip(cusip)); + } + + // ---- The constructed US ISIN ---------------------------------------------------------- + + [TestCase("037833100", "US0378331005")] // Apple + [TestCase("02079K305", "US02079K3059")] // Alphabet Class C + public void TheConstructedIsinMatchesTheRealOne(string cusip, string isin) + { + // This is the second step of the identity chain and it is pure arithmetic, so it is + // either exactly right or it silently resolves to nothing. It lifted measured coverage + // from 76.1% to 88.9% of reported value, which is only true while it is exact: both of + // these are checked against the security's real published ISIN. + Assert.AreEqual(isin, SEC13FDownloader.BuildUnitedStatesIsin(cusip)); + } + + // ---- The VALUE unit break at 2023 ------------------------------------------------------ + + [TestCase("2022-11-14", 1000000)] // reported in thousands, scaled up + [TestCase("2022-12-31", 1000000)] // the last day of the old unit + [TestCase("2023-01-01", 1000)] // the first day of the new one, left alone + [TestCase("2023-02-14", 1000)] + public void ValueIsScaledOnlyForFilingsMadeBefore2023(string filingDate, decimal expected) + { + // VALUE changed unit with the 2023 filings: through 2022 it is thousands of dollars, + // from 2023 whole dollars. Measured on the real archives as the median VALUE/SSHPRNAMT + // of share lines, which is a price per share: 0.0460 in NOV-2022 against 39.86 in + // FEB-2023. Left unscaled this puts a 1000x step in the middle of the published history + // and breaks every comparison that crosses it. The cut is on FILING_DATE rather than on + // the reported period, because the rule changed for filings made from January 2023. + var filing = DateTime.Parse(filingDate); + var holdings = ReadInfoTable( + new[] { Line("0000000000-00-000001", "037833100", "1000", "100", "SH") }, + Submission("0000000000-00-000001", filing, new DateTime(2022, 9, 30))); + + Assert.AreEqual(1, holdings.Count); + Assert.AreEqual(expected, holdings.Values.Single().HoldingValue); + } + + [Test] + public void AFilerThatReportedDollarsBefore2023IsNotScaled() + { + // The rule was thousands, but not every filer followed it. In Apple's December 2019 + // quarter 85 of 5,365 lines were already in dollars, and multiplied by a thousand they + // made 88 percent of the total: an implied $2,577 a share against a $293.65 close. Each + // line is held against the quarter-end close, which needs no other filer. + SeedCloses("20220930", ("aapl", 150m)); + var filing = new DateTime(2022, 11, 14); + var quarter = new DateTime(2022, 9, 30); + var holdings = ReadInfoTable( + new[] + { + Line("0000000000-00-000001", "037833100", "15", "100", "SH"), + Line("0000000000-00-000002", "037833100", "15000", "100", "SH") + }, + resolve: true, + Submission("0000000000-00-000001", filing, quarter, cik: 111), + Submission("0000000000-00-000002", filing, quarter, cik: 222)); + + Assert.AreEqual(15000m + 15000m, holdings.Values.Single().HoldingValue, + "the one in thousands is scaled, the one already in dollars is not"); + } + + [Test] + public void AFilerStillReportingThousandsAfter2023IsScaled() + { + // The same break from the other side: 548 of Apple's March 2026 lines were still in + // thousands, which left the published value five percent short. The quarter ends on a + // Sunday, so the close is the Friday's. + SeedCloses("20231229", ("aapl", 150m)); + var filing = new DateTime(2024, 2, 14); + var quarter = new DateTime(2023, 12, 31); + var holdings = ReadInfoTable( + new[] + { + Line("0000000000-00-000001", "037833100", "15000", "100", "SH"), + Line("0000000000-00-000002", "037833100", "15", "100", "SH") + }, + resolve: true, + Submission("0000000000-00-000001", filing, quarter, cik: 111), + Submission("0000000000-00-000002", filing, quarter, cik: 222)); + + Assert.AreEqual(2 * 15000m, holdings.Values.Single().HoldingValue); + } + + [Test] + public void TheUnitIsDecidedWithoutLookingAtAnyOtherFiling() + { + // The unit used to come from the median of every filing in the three month window, so a + // filing made on its first day was corrected with filings made weeks later. Read alone, + // the way the daily job reads a day, the same filing comes out the same. + SeedCloses("20220930", ("aapl", 150m)); + var filing = new DateTime(2022, 10, 3); + var quarter = new DateTime(2022, 9, 30); + + var alone = ReadInfoTable( + new[] { Line("0000000000-00-000001", "037833100", "15000", "100", "SH") }, + resolve: true, + Submission("0000000000-00-000001", filing, quarter, cik: 111)); + + Assert.AreEqual(15000m, alone.Values.Single().HoldingValue, + "a lone filing in dollars before 2023 is not multiplied by the thousands rule"); + } + + [Test] + public void ALineInAnotherUnitThanTheRestOfItsFilingIsCorrectedOnItsOwn() + { + // Some filers mix units inside one filing, so a decision per filing left Apple's June 2020 + // quarter 14 percent high: the filing is in thousands, its Apple line in dollars. Held + // against its own close, the line is measured on its own. + SeedCloses("20200630", ("aapl", 360m), ("msft", 200m), ("nvda", 400m)); + var filing = new DateTime(2020, 8, 14); + var quarter = new DateTime(2020, 6, 30); + + var holdings = ReadInfoTable( + new[] + { + Line("0000000000-00-000099", "037833100", "36000", "100", "SH"), + Line("0000000000-00-000099", "594918104", "20", "100", "SH"), + Line("0000000000-00-000099", "67066G104", "40", "100", "SH") + }, + resolve: true, + Submission("0000000000-00-000099", filing, quarter, cik: 99)); + + Assert.AreEqual(36000m, holdings.Single(pair => pair.Key.Cusip == "037833100").Value.HoldingValue, + "the dollar line is not scaled"); + Assert.AreEqual(20000m, holdings.Single(pair => pair.Key.Cusip == "594918104").Value.HoldingValue, + "and the rest of its filing still is"); + } + + [Test] + public void ALineAThousandTimesTheCloseIsBroughtDown() + { + // Three SPY lines of the March 2023 quarter carried VALUE a thousand times the price, and 16 + // percent of the total with it. It is the VALUE that is off, not the share count: the same + // filers reported the same number of shares the quarter before, as 4,578 of the 4,740 + // comparable lines of that quarter did. + SeedCloses("20230331", ("spy", 409.39m)); + var filing = new DateTime(2023, 5, 15); + var quarter = new DateTime(2023, 3, 31); + + var holdings = ReadInfoTable( + new[] + { + Line("0000000000-00-000001", "78462F103", "40939", "100", "SH"), + Line("0000000000-00-000002", "78462F103", "40939000", "100", "SH") + }, + resolve: true, + Submission("0000000000-00-000001", filing, quarter, cik: 1), + Submission("0000000000-00-000002", filing, quarter, cik: 2)); + + Assert.AreEqual(2 * 40939m, holdings.Values.Single().HoldingValue); + } + + [Test] + public void ARunWithoutTheSecUserAgentKeysFailsBeforeAnyRequest() + { + // The SEC asks automated readers to identify themselves, and the reports dataset reads the + // name and email from these two keys. Without them the run stops instead of calling out + // anonymously. + var previousName = Config.Get("sec-user-agent-company-name", null); + var previousEmail = Config.Get("sec-user-agent-company-email", null); + try + { + Config.Set("sec-user-agent-company-name", string.Empty); + Config.Set("sec-user-agent-company-email", string.Empty); + + using var downloader = Downloader(); + Assert.Throws(() => downloader.Run()); + } + finally + { + // As in TearDown: written back as an empty string, an absent key would stay set. + Config.Reset(); + if (previousName != null) + { + Config.Set("sec-user-agent-company-name", previousName); + } + + if (previousEmail != null) + { + Config.Set("sec-user-agent-company-email", previousEmail); + } + } + } + + [Test] + public void ALineAMillionTimesOffIsLeftToItsFiling() + { + // A price a thousand times below the close in a thousands filing would take a millionfold + // factor. That is a wrong share count, not a unit, and scaling such lines put Apple's + // December 2019 quarter ten percent above its close. + SeedCloses("20191231", ("aapl", 290m)); + var filing = new DateTime(2020, 2, 14); + var quarter = new DateTime(2019, 12, 31); + + var holdings = ReadInfoTable( + new[] + { + Line("0000000000-00-000001", "037833100", "29", "100", "SH"), + Line("0000000000-00-000002", "037833100", "29", "100", "SH"), + Line("0000000000-00-000002", "037833100", "29", "100", "SH"), + Line("0000000000-00-000002", "037833100", "29", "100000", "SH") + }, + resolve: true, + Submission("0000000000-00-000001", filing, quarter, cik: 1), + Submission("0000000000-00-000002", filing, quarter, cik: 2)); + + Assert.AreEqual(4 * 29000m, holdings.Values.Single().HoldingValue, + "the odd line keeps the thousands factor its other lines prove"); + } + + [TestCase(true, 1000, 0, new double[0])] // nothing to compare against: the rule stands + [TestCase(false, 1, 0, new double[0])] + [TestCase(true, 1000, 3, new[] { -3.0, -2.9, -3.1 })] // thousands, as the rule says + [TestCase(true, 1, 4, new[] { 0.0, 0.1, -0.1, -3.0 })] // whole dollars before 2023 + [TestCase(false, 1, 2, new[] { 0.1, -0.1 })] + [TestCase(false, 1000, 3, new[] { -3.0, -2.8, 0.1 })] // still thousands after 2023 + [TestCase(false, 1, 10, new[] { -3.0, -2.9 })] // two of ten priced lines on a step: no evidence + [TestCase(true, 1, 10, new[] { -3.0, -2.9 })] // and before 2023 it is not scaled up either + public void AFilingsUnitComesFromHowItsPricesCompareWithTheClose(bool thousandsRule, + decimal expected, int priced, double[] offsets) + { + Assert.AreEqual(expected, SEC13FDownloader.ValueMultiplier(offsets.ToList(), priced, thousandsRule)); + } + + [Test] + public void AFilingWithTheShareCountTypedAsValueIsNotScaled() + { + // A manager's 9 March 2026 filing carried SSHPRNAMT equal to VALUE on every line, a price of + // exactly $1. Against a close near $1,000 that reads as thousands, so the filing was taken to + // report in thousands and every one of its lines was multiplied: Dow's December 2025 quarter + // went from $12.6 billion to $50.2 billion. Most of its lines sit on no unit step, which is no + // evidence of a unit, and a line that sits on none keeps the rule of its filing date. + SeedCloses("20251231", ("aapl", 250m), ("msft", 480m), ("nvda", 180m), ("spy", 680m)); + var filing = new DateTime(2026, 3, 9); + var quarter = new DateTime(2025, 12, 31); + var holdings = ReadInfoTable( + new[] + { + Line("0000000000-00-000001", "037833100", "1000000", "1000000", "SH"), + Line("0000000000-00-000001", "594918104", "500000", "500000", "SH"), + Line("0000000000-00-000001", "67066G104", "300000", "300000", "SH"), + Line("0000000000-00-000001", "78462F103", "200000", "200000", "SH") + }, + resolve: true, + Submission("0000000000-00-000001", filing, quarter, cik: 1964189)); + + Assert.AreEqual(1000000m, holdings.Single(pair => pair.Key.Cusip == "037833100").Value.HoldingValue); + Assert.AreEqual(300000m, holdings.Single(pair => pair.Key.Cusip == "67066G104").Value.HoldingValue); + + // MSFT at $480 and SPY at $680 put a $1 price on the thousands step by chance. In a filing + // whose lines mostly sit on no step those landings are not evidence either. + Assert.AreEqual(500000m, holdings.Single(pair => pair.Key.Cusip == "594918104").Value.HoldingValue); + Assert.AreEqual(200000m, holdings.Single(pair => pair.Key.Cusip == "78462F103").Value.HoldingValue); + } + + [Test] + public void PrincipalValueIsThePrincipalAmountAndNotTheMarketValue() + { + // On a PRN line SSHPRNAMT is the principal amount and VALUE what that debt is worth. + // The property publishes the first, as its documentation says. + var holdings = ReadInfoTable( + new[] { Line("0000000000-00-000001", "037833100", "4800000", "5000000", "PRN") }, + Submission("0000000000-00-000001", new DateTime(2024, 2, 14), new DateTime(2023, 12, 31))); + + var holding = holdings.Values.Single(); + Assert.AreEqual(5000000m, holding.PrincipalValue); + Assert.AreEqual(0m, holding.HoldingValue, "debt stays out of the share value"); + } + + [Test] + public void AnAmendmentIsKeptApartFromTheOriginalsUntilItsFilerIsKnown() + { + // The reader cannot tell whether an amendment restates a position the manager already + // reported, possibly in an earlier archive, so it keeps each amending filer's lines apart + // instead of summing them in. AdmitAmendment decides later, against the filer state. + var filing = new DateTime(2024, 2, 14); + var holdings = ReadInfoTable( + new[] + { + Line("0000000000-00-000001", "037833100", "1000", "100", "SH"), + Line("0000000000-00-000002", "037833100", "2000", "300", "SH") + }, + Submission("0000000000-00-000001", filing, new DateTime(2023, 12, 31), cik: 111), + Submission("0000000000-00-000002", filing, new DateTime(2023, 12, 31), cik: 222, isAmendment: true)); + + var holding = holdings.Values.Single(); + + Assert.AreEqual(new[] { 111 }, holding.Ciks.ToArray(), "only the original's filer is gathered"); + Assert.AreEqual(100m, holding.Shares, "only the original's shares are in the totals"); + Assert.AreEqual(1000m, holding.HoldingValue, "only the original's value is in the totals"); + Assert.AreEqual(300m, holding.Amendments[222].Shares, "the amendment's shares are held apart"); + Assert.AreEqual(2000m, holding.Amendments[222].HoldingValue, "the amendment's value is held apart"); + } + + [Test] + public void ARestatementOfAPositionAlreadyReportedAddsNothing() + { + // The double count this rule removes. Apple's March 2026 quarter carried 61 managers with + // both an original and a RESTATEMENT, and summing both put 993,574,019 shares in twice, + // 9.6 percent of the published total. Once the manager is counted for the security and + // quarter, its later amendment is not added on top. + using var downloader = Downloader(); + var security = Security("AAPL"); + var quarter = new DateTime(2023, 12, 31); + + downloader.CountNewFilers(security, quarter, new[] { 111 }, new DateTime(2024, 2, 1)); + + Assert.IsFalse(downloader.AdmitAmendment(security, quarter, 111, new DateTime(2024, 3, 1), out var isNew), + "a restatement of a position the manager already reported must not be summed"); + Assert.IsFalse(isNew, "and the manager is not counted again"); + } + + [Test] + public void AnAmendmentNamingASecurityForTheFirstTimeIsAdded() + { + // What a NEW HOLDINGS amendment carries, and what a restatement carries for a security the + // original left out: a position nobody had counted for this manager yet. It is added, and + // the manager becomes a holder. + using var downloader = Downloader(); + var security = Security("AAPL"); + var quarter = new DateTime(2023, 12, 31); + + Assert.IsTrue(downloader.AdmitAmendment(security, quarter, 222, new DateTime(2024, 3, 1), out var isNew)); + Assert.IsTrue(isNew, "the manager is new to this security and quarter"); + } + + [Test] + public void OneAmendmentNamingTwoCusipsOfTheSameSecurityIsAddedForBoth() + { + // An issuer's share classes, or a debt CUSIP, resolve to the same security. The first + // CUSIP's group marks the manager counted, and the second group of the same filing on + // the same day must still be admitted, or half of that amendment would vanish. + using var downloader = Downloader(); + var security = Security("AAPL"); + var quarter = new DateTime(2023, 12, 31); + var filed = new DateTime(2024, 3, 1); + + Assert.IsTrue(downloader.AdmitAmendment(security, quarter, 333, filed, out var first)); + Assert.IsTrue(downloader.AdmitAmendment(security, quarter, 333, filed, out var second), + "the same amendment's second CUSIP is part of the same position"); + Assert.IsTrue(first, "counted on the first CUSIP"); + Assert.IsFalse(second, "and only once"); + Assert.IsFalse(downloader.AdmitAmendment(security, quarter, 333, filed.AddDays(10), out _), + "a later amendment by the same manager is a restatement of what it already added"); + } + + // ---- Holders is a distinct count of managers -------------------------------------------- + + [Test] + public void AFilerReachesHoldersOnceHoweverManyTimesItReportsTheQuarter() + { + // Measured against the source for Bristol-Myers Squibb's December 2019 quarter: summing + // the per filing counts gives 2,765 managers where 2,235 distinct ones reported, because + // 530 of them named both the common stock and the contingent value right, 525 of those + // on one filing. Summing counts a manager once per CUSIP and once per day it appears. + using var downloader = Downloader(); + var security = Security("AAPL"); + var quarter = new DateTime(2023, 12, 31); + + Assert.AreEqual(3, downloader.CountNewFilers(security, quarter, new[] { 111, 222, 333 }, Filed), + "three managers, first sight"); + Assert.AreEqual(0, downloader.CountNewFilers(security, quarter, new[] { 111, 222 }, Filed), + "the same managers on a second CUSIP or a later day add nothing"); + Assert.AreEqual(1, downloader.CountNewFilers(security, quarter, new[] { 222, 444 }, Filed), + "only the manager that had not reported yet is added"); + } + + [Test] + public void AManagerThatNamesTheSecurityOnlyInAnAmendmentIsCounted() + { + // The undercount the earlier rule had. A manager that leaves a security out of its + // original report and adds it in the amendment holds that security, and its shares and + // value are carried, so leaving it out published value with no holder at all. Apple has + // eight such managers in the March 2026 quarter. + using var downloader = Downloader(); + var security = Security("AAPL"); + var quarter = new DateTime(2023, 12, 31); + + downloader.CountNewFilers(security, quarter, new[] { 111 }, Filed); + + Assert.AreEqual(1, downloader.CountNewFilers(security, quarter, new[] { 222 }, Filed), + "the amendment's filer is new to this security and quarter"); + } + + [Test] + public void FilersAreCountedPerSecurityAndPerQuarter() + { + // The key has to carry all three parts. A manager counted for one quarter must still be + // new to the next one, which is what makes each quarter its own census rather than a + // running total of every quarter before it. + using var downloader = Downloader(); + var apple = Security("AAPL"); + var microsoft = Security("MSFT"); + + downloader.CountNewFilers(apple, new DateTime(2023, 12, 31), new[] { 111 }, Filed); + + Assert.AreEqual(1, downloader.CountNewFilers(apple, new DateTime(2024, 3, 31), new[] { 111 }, Filed), + "a new quarter counts the manager again"); + Assert.AreEqual(1, downloader.CountNewFilers(microsoft, new DateTime(2023, 12, 31), new[] { 111 }, Filed), + "a different security counts the manager again"); + } + + [Test] + public void APeriodThatIsNotAQuarterEndGetsItsOwnSlot() + { + // PERIODOFREPORT is free text from the filer. Deriving the key from the quarter a date + // falls in would fold a malformed period into a real one and credit its filers there. + using var downloader = Downloader(); + var security = Security("AAPL"); + + downloader.CountNewFilers(security, new DateTime(2023, 12, 31), new[] { 111 }, Filed); + + Assert.AreEqual(1, downloader.CountNewFilers(security, new DateTime(2023, 12, 15), new[] { 111 }, Filed), + "a period inside the same quarter is still a different period"); + } + + // ---- Accumulation ---------------------------------------------------------------------- + + [Test] + public void IncrementsRunningSumWithinAQuarterAndNeverAcrossTwo() + { + // The managers of one quarter file across roughly fifty days, so the published row states + // what had been reported for its own quarter as of its own release date. Carrying the + // running total into the next quarter would restate a fresh quarter as the sum of every + // quarter before it. + var cumulative = SEC13FDownloader.Accumulate(new[] + { + Row("20240201 17:30", "20231231", 3m, 1000.25m), + Row("20240214 17:30", "20231231", 2m, 500.75m), + Row("20240415 17:30", "20240331", 1m, 7.5m) + }); + + Assert.AreEqual(new[] { 3m, 5m, 1m }, cumulative.Select(row => row.Values[0]).ToArray()); + Assert.AreEqual(new[] { 1000.25m, 1501m, 7.5m }, cumulative.Select(row => row.Values[1]).ToArray()); + } + + [Test] + public void TheConfidentialFlagOnlyEverGoesUpWithinAQuarter() + { + // Once a contributing filing withheld positions the quarter's total is incomplete and + // stays incomplete, however many complete filings arrive after it. The next quarter is + // a fresh total, so it starts clean again. + var cumulative = SEC13FDownloader.Accumulate(new[] + { + Row("20240201 17:30", "20231231", 1m, 10m), + Row("20240214 17:30", "20231231", 1m, 10m, confidential: true), + Row("20240301 17:30", "20231231", 1m, 10m), + Row("20240415 17:30", "20240331", 1m, 10m) + }); + + Assert.AreEqual(new[] { false, true, true, false }, + cumulative.Select(row => row.ConfidentialOmitted).ToArray()); + } + + // ---- Difference and re-accumulate ------------------------------------------------------- + + [Test] + public void DifferencingACumulativeFileRecoversTheIncrementsExactly() + { + // This is the property that makes an incremental run idempotent, and the one that would + // break in silence. A published file holds running totals, and two running totals over + // different sets of filings cannot be combined at all, so the merge differences the + // published copy back to increments first. Everything is decimal on purpose: the + // difference is the exact inverse of the sum only while no rounding happens, so the + // values here are deliberately awkward rather than round. + var increments = new[] + { + Row("20240201 17:30", "20231231", 3m, 1234.56789m), + Row("20240214 17:30", "20231231", 2m, 0.00000001m), + Row("20240301 17:30", "20231231", 7m, 98765.4321m, confidential: true), + Row("20240415 17:30", "20240331", 1m, 0.1m), + Row("20240515 17:30", "20240331", 4m, 0.2m) + }; + + var roundTripped = SEC13FDownloader + .Difference(SEC13FDownloader.Accumulate(increments)) + .ToDictionary(row => (row.PeriodEnd, row.Time)); + + Assert.AreEqual(increments.Length, roundTripped.Count); + + foreach (var original in increments) + { + var recovered = roundTripped[(original.PeriodEnd, original.Time)]; + + Assert.AreEqual(original.Values, recovered.Values, + $"{original.PeriodEnd:yyyyMMdd} at {original.Time:yyyyMMdd HH:mm} did not round-trip"); + Assert.AreEqual(original.ConfidentialOmitted || recovered.ConfidentialOmitted, + recovered.ConfidentialOmitted, "the flag is OR-ed forward, so it never comes back down"); + } + } + + [Test] + public void AMergeThatReadsNoNewFilingReproducesThePublishedRows() + { + // The same property as above, seen from where it actually matters: the production job + // runs daily and most days bring nothing new for a given security. What it publishes then + // has to be what it read, value for value, or every quiet day would perturb the history. + var published = new (string, SEC13FDownloader.HoldingsRow)[] + { + ("aapl", Row("20240201 17:30", "20231231", 3m, 1234.56789m)), + ("aapl", Row("20240214 17:30", "20231231", 5m, 1234.56789001m, confidential: true)) + }; + + var merged = SEC13FDownloader.MergeSecurity(published, Array.Empty<(string, SEC13FDownloader.HoldingsRow)>()); + + Assert.AreEqual(published.Length, merged.Count); + for (var i = 0; i < published.Length; i++) + { + Assert.AreEqual(published[i].Item1, merged[i].Ticker); + Assert.AreEqual(published[i].Item2.Time, merged[i].Row.Time); + Assert.AreEqual(published[i].Item2.Values, merged[i].Row.Values); + Assert.AreEqual(published[i].Item2.ConfidentialOmitted, merged[i].Row.ConfidentialOmitted); + } + } + + [Test] + public void ARenameCarriesTheQuartersRunningTotalIntoTheNewTicker() + { + // The defect this exists for. Facebook became Meta on 2022-06-09, while the March 2022 + // quarter was still receiving late filings. With running totals kept per ticker file, + // meta.csv restarted that quarter at zero and the universe showed Meta held by 3 managers + // the week after it had shown 3,182. The total belongs to the security, and carries on + // across the rename in whichever file the row lands. + var published = new (string, SEC13FDownloader.HoldingsRow)[] + { + ("fb", Row("20220401 17:30", "20220331", 2m, 204644m)), + ("fb", Row("20220608 17:30", "20220331", 3182m, 1827789746m)) + }; + var increments = new (string, SEC13FDownloader.HoldingsRow)[] + { + ("meta", Row("20220610 17:30", "20220331", 0m, 30693m)), + ("meta", Row("20220613 17:30", "20220331", 2m, 21012m)) + }; + + var merged = SEC13FDownloader.MergeSecurity(published, increments); + + Assert.AreEqual(new[] { "fb", "fb", "meta", "meta" }, merged.Select(entry => entry.Ticker).ToArray()); + Assert.AreEqual(3182m, merged[2].Row.Values[0], "the quarter's holders carry on into meta.csv"); + Assert.AreEqual(1827789746m + 30693m, merged[2].Row.Values[1], "and so do its shares"); + Assert.AreEqual(3184m, merged[3].Row.Values[0]); + } + + [Test] + public void ThisRunWinsOverThePublishedReadingOfTheSameReleaseDate() + { + // An incremental run reads its window again, so its reading of a quarter and release date + // is the current one and replaces the published increment rather than adding to it. + var published = new (string, SEC13FDownloader.HoldingsRow)[] + { + ("aapl", Row("20240201 17:30", "20231231", 3m, 30m)), + ("aapl", Row("20240214 17:30", "20231231", 5m, 50m)) + }; + var increments = new (string, SEC13FDownloader.HoldingsRow)[] + { + ("aapl", Row("20240214 17:30", "20231231", 4m, 40m)) + }; + + var merged = SEC13FDownloader.MergeSecurity(published, increments); + + Assert.AreEqual(2, merged.Count); + Assert.AreEqual(7m, merged[1].Row.Values[0], "three published plus the four re-read, not plus the two published"); + Assert.AreEqual(70m, merged[1].Row.Values[1]); + } + + // ---- Chronological output order --------------------------------------------------------- + + [Test] + public void MergedRowsComeOutInReleaseOrderAndNotQuarterOrder() + { + // Accumulate() groups by quarter, so its natural order is NOT the order the file ships + // in, and the two disagree exactly when a late amendment for an old quarter arrives + // after a newer quarter has started reporting. LEAN's SubscriptionDataReader drops a + // point whose timestamp moves backwards and does so silently, so a file left in quarter + // order loses rows with nothing in the log to show for it. + var published = new (string, SEC13FDownloader.HoldingsRow)[] + { + ("aapl", Row("20240215 17:30", "20231231", 1m, 5m)) + }; + var increments = new (string, SEC13FDownloader.HoldingsRow)[] + { + // Newer quarter, released first. + ("aapl", Row("20240501 17:30", "20240331", 5m, 100m)), + // Older quarter, an amendment released two weeks later. + ("aapl", Row("20240515 17:30", "20231231", 1m, 7m)) + }; + + var merged = SEC13FDownloader.MergeSecurity(published, increments); + + Assert.AreEqual(new[] { "20240215 17:30", "20240501 17:30", "20240515 17:30" }, + merged.Select(entry => entry.Row.Time.ToStringInvariant("yyyyMMdd HH:mm")).ToArray()); + + // The amendment accumulates onto the published row of its own quarter rather than + // starting over: one holder plus one, five shares plus seven. + Assert.AreEqual(2m, merged[2].Row.Values[0]); + Assert.AreEqual(12m, merged[2].Row.Values[1]); + } + + // ---- The incremental run has to see the published history --------------------------------- + + [Test] + public void AnIncrementalRunWithNoPublishedHistoryFails() + { + // The destination arrives empty on every deployment, so a processed-data-directory that + // is wrong or unmounted leaves an incremental run with nothing to merge into. Publishing + // the window on its own would republish thirteen years as three months, rebuild every + // universe file from it, and return success: the files come out the right shape, so no + // row count tells the two apart. The guard runs before anything is fetched. + using var downloader = new SEC13FDownloader( + Path.Combine(_root, "out"), Path.Combine(_root, "processed"), new DateTime(2026, 9, 8)); + + Assert.Throws(() => downloader.Run(), + "the run carried on with no history behind it"); + } + + [Test] + public void ARunIntoADestinationAlreadyHoldingFilesFails() + { + // The job hands the destination over empty. Files an earlier run left there would be read + // into the universe and published again, so a run refuses to start on top of them. + var destination = Path.Combine(_root, "out", SEC13F.ReportFolder); + Directory.CreateDirectory(destination); + File.WriteAllText(Path.Combine(destination, "aapl.csv"), "20240214 17:30,1,20231231,1,1,1,0,0,0,0,0,0"); + + using var downloader = Downloader(); + + Assert.Throws(() => downloader.RequireEmptyDestination()); + } + + // ---- Publication time and the EDGAR days --------------------------------------------------- + + [Test] + public void AFilingIsPublishedTheMorningAfterItsFilingDate() + { + // The defect this dataset shipped with: a row stamped 17:30 on its filing date, when the + // SEC data sets publish that filing up to three months later. EDGAR lists a day's filings + // at about 22:05 ET and the daily job reads them at 01:00 the next day, so the row is + // published at 03:00 the morning after, and a backtest can read it no earlier. + using var downloader = Downloader(); + + Assert.AreEqual(new DateTime(2026, 5, 11, 3, 0, 0), downloader.ReleaseOf(new DateTime(2026, 5, 10))); + } + + [Test] + public void ADayReadLateIsPublishedWhenTheRunReadIt() + { + // A daily run that picks up a day it missed, a late index or a failed night, could not + // have published that day before its own deployment date. Stamping the rows with the + // filing date would put them in the past, where live never delivered them. + using var downloader = new SEC13FDownloader( + Path.Combine(_root, "out"), Path.Combine(_root, "processed"), new DateTime(2026, 9, 9)); + + Assert.AreEqual(new DateTime(2026, 9, 10, 3, 0, 0), downloader.ReleaseOf(new DateTime(2026, 9, 3)), + "the missed day is stamped with the run that read it"); + Assert.AreEqual(new DateTime(2026, 9, 10, 3, 0, 0), downloader.ReleaseOf(new DateTime(2026, 9, 9)), + "and the deployment date itself the usual way"); + } + + [Test] + public void TheRebuildHandsOverToEdgarTheDayAfterTheLastDataSet() + { + // The data sets come out in three month batches, so the rebuild reads them as far as they + // reach and EDGAR after that, the way the daily job does. EDGAR can also take over + // earlier, which is how the two sources are compared over a window both carry. + var archives = new List + { + new("01dec2025-28feb2026_form13f.zip", "https://localhost/a.zip", new DateTime(2025, 12, 1), new DateTime(2026, 2, 28)), + new("01mar2026-31may2026_form13f.zip", "https://localhost/b.zip", new DateTime(2026, 3, 1), new DateTime(2026, 5, 31)) + }; + + Assert.AreEqual(new[] { "01dec2025-28feb2026_form13f.zip", "01mar2026-31may2026_form13f.zip" }, + SEC13FDownloader.ArchivesBefore(archives, new DateTime(2026, 6, 1)).Select(archive => archive.Name).ToArray()); + Assert.AreEqual(new[] { "01dec2025-28feb2026_form13f.zip" }, + SEC13FDownloader.ArchivesBefore(archives, new DateTime(2026, 3, 1)).Select(archive => archive.Name).ToArray()); + } + + [Test] + public void EdgarCannotTakeOverInTheMiddleOfADataSet() + { + // The window's filings before the switch would come from the data set and the rest from + // EDGAR only if the archive were cut by date, which it is not: it would be read twice. + var archives = new List + { + new("01mar2026-31may2026_form13f.zip", "https://localhost/b.zip", new DateTime(2026, 3, 1), new DateTime(2026, 5, 31)) + }; + + Assert.Throws(() => SEC13FDownloader.ArchivesBefore(archives, new DateTime(2026, 4, 15))); + } + + [Test] + public void ADailyRunReadsTheWeekdaysNoEarlierRunFoldedIn() + { + // A day already published is never read again, since its filings would be added a second + // time under a later stamp, and a day without an index is tried again by the next run. + // Nor is a day before EDGAR took over from the data sets: the rebuild read those from the + // data set, so they are not in the list, and reading them from EDGAR counted them twice. + var shelf = ShelfWithFilerState(); + File.WriteAllText(Path.Combine(shelf, "edgar-days.txt"), "#from 20260901\n20260903\n20260904\n"); + + using var downloader = new SEC13FDownloader( + Path.Combine(_root, "out"), Path.Combine(_root, "processed"), new DateTime(2026, 9, 9)); + downloader.ReadEdgarState(); + + Assert.AreEqual( + new[] { "20260901", "20260902", "20260907", "20260908", "20260909" }, + downloader.EdgarDaysToRead(new DateTime(2026, 8, 30), new DateTime(2026, 9, 9)) + .Select(day => day.ToString("yyyyMMdd")).ToArray()); + } + + [Test] + public void AnIncrementalRunWithoutTheEdgarStateFails() + { + // Without the list of days already published a run cannot tell a new day from one it + // would add a second time, so it stops instead of guessing. + var shelf = ShelfWithFilerState(); + File.Delete(Path.Combine(shelf, "edgar-days.txt")); + File.WriteAllLines(Path.Combine(shelf, "aapl.csv"), new[] { "20240215 03:00,1,20231231,1,5,50,0,0,0,0,0,0" }); + + using var downloader = new SEC13FDownloader( + Path.Combine(_root, "out"), Path.Combine(_root, "processed"), new DateTime(2026, 9, 9)); + + Assert.Throws(() => downloader.RequirePublishedHistoryForIncrementalRun()); + } + + [Test] + public void AMissingDeploymentDateIsAnErrorUnlessTheRebuildIsAskedFor() + { + // An empty date used to mean the full history, which turned a misconfigured nightly job + // into a five gigabyte refetch that exited zero. + var previousDate = Environment.GetEnvironmentVariable("QC_DATAFLEET_DEPLOYMENT_DATE"); + try + { + Environment.SetEnvironmentVariable("QC_DATAFLEET_DEPLOYMENT_DATE", null); + + Config.Set(Program.RebuildHistoryKey, "false"); + Assert.IsFalse(Program.TryParseDeploymentDate(out _), "no date and no rebuild asked for"); + + Config.Set(Program.RebuildHistoryKey, "true"); + Assert.IsTrue(Program.TryParseDeploymentDate(out var rebuild)); + Assert.IsNull(rebuild, "the rebuild runs over the whole history"); + + Environment.SetEnvironmentVariable("QC_DATAFLEET_DEPLOYMENT_DATE", "20260908"); + Assert.IsTrue(Program.TryParseDeploymentDate(out var date)); + Assert.AreEqual(new DateTime(2026, 9, 8), date); + } + finally + { + Environment.SetEnvironmentVariable("QC_DATAFLEET_DEPLOYMENT_DATE", previousDate); + Config.Set(Program.RebuildHistoryKey, "false"); + } + } + + [Test] + public void AnIncrementalRunAcceptsAShelfThatHoldsSecurities() + { + // The same guard from the other side: a published history present is not a failure. Only + // the guard is exercised here, by handing it a deployment date and a shelf and checking + // it does not stop the run before the first archive is fetched. + var processed = ShelfWithFilerState(); + File.WriteAllLines(Path.Combine(processed, "aapl.csv"), new[] + { + "20240215 17:30,1,20231231,1,5,50,0,0,0,0,0,0" + }); + + using var downloader = new SEC13FDownloader( + Path.Combine(_root, "out"), Path.Combine(_root, "processed"), new DateTime(2026, 9, 8)); + + Assert.DoesNotThrow(() => downloader.RequirePublishedHistoryForIncrementalRun()); + } + + [Test] + public void AnIncrementalRunWithAHistoryButNoFilerStateFails() + { + // Holders is a distinct count and cannot be rebuilt from the published totals the way + // the summed columns can, so a run that cannot read the state would count every filer in + // tonight's archive again on top of the existing history. The file would keep its shape + // and only that one column would be wrong, which is what this refuses to publish. + var processed = Path.Combine(_root, "processed", SEC13F.ReportFolder); + Directory.CreateDirectory(processed); + File.WriteAllLines(Path.Combine(processed, "aapl.csv"), new[] + { + "20240215 17:30,1,20231231,1,5,50,0,0,0,0,0,0" + }); + + using var downloader = new SEC13FDownloader( + Path.Combine(_root, "out"), Path.Combine(_root, "processed"), new DateTime(2026, 9, 8)); + + Assert.Throws( + () => downloader.RequirePublishedHistoryForIncrementalRun(), + "a shelf with securities but no filer state has to stop the run"); + } + + [Test] + public void AFullHistoryRunNeedsNoFilerState() + { + // A full run rebuilds every count from the archives, so it neither needs the state nor + // reads it. Requiring it there would make the one run that can restore the file + // impossible whenever the file is what went missing. + using var downloader = Downloader(); + + Assert.DoesNotThrow(() => downloader.RequirePublishedHistoryForIncrementalRun()); + Assert.DoesNotThrow(() => downloader.ReadFilerState()); + } + + [Test] + public void TheFilerStateSurvivesAWriteAndReadRoundTrip() + { + var security = Security("AAPL"); + var other = Security("MSFT"); + var quarter = new DateTime(2023, 12, 31); + + using (var first = Downloader()) + { + first.CountNewFilers(security, quarter, new[] { 111, 222, 333 }, Filed); + first.CountNewFilers(other, quarter, new[] { 111 }, Filed); + first.WriteFilerState(); + } + + // The shelf is what the next run reads and the publish step is what puts this run's + // output there, so the round trip goes through that move rather than reading the output + // in place. + var shelf = Path.Combine(_root, "processed", SEC13F.ReportFolder); + Directory.CreateDirectory(shelf); + File.Copy( + Path.Combine(_root, "out", SEC13F.ReportFolder, "filers.zip"), + Path.Combine(shelf, "filers.zip"), overwrite: true); + + using var second = new SEC13FDownloader( + Path.Combine(_root, "out"), Path.Combine(_root, "processed"), new DateTime(2024, 2, 14)); + second.ReadFilerState(); + + Assert.AreEqual(0, second.CountNewFilers(security, quarter, new[] { 111, 222, 333 }, Filed), + "every filer the earlier run counted is already known"); + Assert.AreEqual(0, second.CountNewFilers(other, quarter, new[] { 111 }, Filed), + "the same manager on another security is known there too"); + Assert.AreEqual(1, second.CountNewFilers(security, quarter, new[] { 444 }, Filed), + "a manager nobody has seen is still new"); + } + + [Test] + public void TheFilerStateIsByteReproducible() + { + // The state ships with the data, so a run that read no new filing has to produce the + // same bytes. A zip stamps every entry with the moment it was written, which would make + // the delivery archive look changed on every run when nothing in it had moved. + var security = Security("AAPL"); + var quarter = new DateTime(2023, 12, 31); + var path = Path.Combine(_root, "out", SEC13F.ReportFolder, "filers.zip"); + + using (var first = Downloader()) + { + first.CountNewFilers(security, quarter, new[] { 111, 222 }, Filed); + first.WriteFilerState(); + } + + var before = File.ReadAllBytes(path); + + using (var second = Downloader()) + { + second.CountNewFilers(security, quarter, new[] { 111, 222 }, Filed); + second.WriteFilerState(); + } + + Assert.AreEqual(Convert.ToBase64String(before), Convert.ToBase64String(File.ReadAllBytes(path)), + "the same filers have to write the same archive"); + } + + [Test] + public void TheFilerStateDoesNotDependOnTheOrderFilersWereFirstSeen() + { + // A full run meets securities and quarters in filing order, while an incremental run + // meets them in the order the published state lists them. Written in the order of the + // packed keys, whose indexes come from that first sighting, the same filers came out in + // a different order: the incremental republished all 58 million of them unchanged and + // filers.zip still changed bytes. + var path = Path.Combine(_root, "out", SEC13F.ReportFolder, "filers.zip"); + var apple = Security("AAPL"); + var microsoft = Security("MSFT"); + var december = new DateTime(2023, 12, 31); + var march = new DateTime(2024, 3, 31); + + using (var first = Downloader()) + { + first.CountNewFilers(microsoft, march, new[] { 222 }, Filed); + first.CountNewFilers(apple, december, new[] { 111 }, Filed); + first.CountNewFilers(apple, march, new[] { 333 }, Filed); + first.WriteFilerState(); + } + + var before = File.ReadAllBytes(path); + + using (var second = Downloader()) + { + second.CountNewFilers(apple, december, new[] { 111 }, Filed); + second.CountNewFilers(apple, march, new[] { 333 }, Filed); + second.CountNewFilers(microsoft, march, new[] { 222 }, Filed); + second.WriteFilerState(); + } + + Assert.AreEqual(Convert.ToBase64String(before), Convert.ToBase64String(File.ReadAllBytes(path)), + "the same filers seen in another order have to write the same archive"); + } + + /// + /// An equity identifier built without touching the map files. These tests are about how + /// filers are counted, not about identity resolution, and the mapping overload reaches for a + /// map file provider that most of them have no fixture for. + /// + private static SecurityIdentifier Security(string ticker) + { + return SecurityIdentifier.GenerateEquity(ticker, Market.USA, mapSymbol: false); + } + + /// A shelf directory carrying a filer state, which an incremental run demands. + private string ShelfWithFilerState() + { + using (var seed = Downloader()) + { + seed.CountNewFilers(Security("AAPL"), + new DateTime(2023, 12, 31), new[] { 1 }, Filed); + seed.WriteFilerState(); + } + + var shelf = Path.Combine(_root, "processed", SEC13F.ReportFolder); + Directory.CreateDirectory(shelf); + File.Copy( + Path.Combine(_root, "out", SEC13F.ReportFolder, "filers.zip"), + Path.Combine(shelf, "filers.zip"), overwrite: true); + File.WriteAllText(Path.Combine(shelf, "edgar-days.txt"), "#from 20260601\n20260601\n"); + return shelf; + } + + // ---- The universe two-quarter rule ------------------------------------------------------- + + [Test] + public void TheUniverseCarriesTheNewestQuarterAndTheOneBeforeIt() + { + // A quarter fills in over roughly fifty days, so on any given day a security has a brand + // new quarter with a handful of managers in it and a finished one with thousands. + // Publishing only the newest throws the finished cross-section away for six weeks of + // every year; publishing only the finished one hides the fresh data. Both are carried, + // and the older is retired the moment the newer overtakes it in Holders, which is the + // point where it stops adding anything. ExtractAlphaTrueBeats ships every live fiscal + // period the same way rather than choosing between them. + var universe = BuildUniverse( + // One release per line: the stamp, how many quarters it restates, and then that + // many groups. Holders is the second column of a group, right after the quarter. + "20231101 17:30,1,20230930,1500,100,1000,0,0,0,0,0,0", + "20240201 17:30,1,20231231,5,10,100,0,0,0,0,0,0", + "20240415 17:30,1,20240331,3,7,70,0,0,0,0,0,0", + "20240516 17:30,1,20240331,9,20,200,0,0,0,0,0,0", + // An amendment for a quarter that has already been retired. + "20240603 17:30,1,20230930,1600,110,1100,0,0,0,0,0,0"); + + // Only the finished quarter exists yet. + Assert.AreEqual(new[] { "20230930" }, universe["20231101"]); + + // The new quarter has five managers against fifteen hundred, so both are published. + Assert.AreEqual(new[] { "20230930", "20231231" }, universe["20240201"]); + + // A third quarter arrives and the oldest goes, whatever its Holders: two is the cap. + Assert.AreEqual(new[] { "20231231", "20240331" }, universe["20240415"]); + + // The newest overtakes the one before it, which is where the older stops being useful. + Assert.AreEqual(new[] { "20240331" }, universe["20240516"]); + + // The late amendment for 2023-09-30 must not put a stale cross-section back into the + // current file. Its own release-date row still exists in the per-security file, so + // nothing is lost by ignoring it here. + Assert.AreEqual(new[] { "20240331" }, universe["20240603"]); + } + + // ---- One line per release, and one line per security --------------------------------------- + + [Test] + public void ASecuritysLiveQuartersTravelInOneUniverseLine() + { + // Both views carry every quarter of a release in a single line, because LEAN hands an + // algorithm one data point per security per timestamp and drops the rest. The per + // security line here restates two quarters at once, and the universe has to unpack it + // into its quarters and pack the live ones back into the security's own line, carrying + // the columns across unchanged so that the two views cannot disagree. + SeedMapFiles("aapl"); + + var holdings = Path.Combine(_root, "out", SEC13F.ReportFolder); + Directory.CreateDirectory(holdings); + File.WriteAllLines(Path.Combine(holdings, "aapl.csv"), new[] + { + "20240201 17:30,2," + + "20230930,1500,100,1000,0,0,0,0,0,1," + + "20231231,5,10,100,0,0,0,0,0,0" + }); + + using (var downloader = Downloader()) + { + downloader.BuildUniverseFiles(); + } + + var lines = File.ReadAllLines(Path.Combine(holdings, "universe", "20240201.csv")) + .Where(line => line.Length > 0) + .ToList(); + + Assert.AreEqual(1, lines.Count, "one line per security, however many quarters it carries"); + + var csv = lines[0].Split(','); + Assert.AreEqual(ListedSince1980("aapl").ToString(), csv[0]); + Assert.AreEqual("AAPL", csv[1]); + Assert.AreEqual("2", csv[2], "both live quarters travel together"); + Assert.AreEqual("20230930,1500,100,1000,0,0,0,0,0,1", string.Join(",", csv.Skip(3).Take(10)), + "the finished quarter, oldest first and column for column"); + Assert.AreEqual("20231231,5,10,100,0,0,0,0,0,0", string.Join(",", csv.Skip(13).Take(10)), + "then the one filling in behind it"); + } + + // ---- Delisted securities and the universe's shelf life ---------------------------------- + + [Test] + public void AFilingAfterTheDelistingHasNoTickerToLandIn() + { + // The map file ends at the delisting, and falling back to the last ticker wrote years of + // later filings under it: DirecTV, delisted in 2015, sat in the May 2026 universe, and a + // ticker reused by another company mixed two companies in one file. After the last map + // file row there is no ticker, and LEAN would not read such a row in any case. + SeedMapFileRows(("dead", new[] { "19801212,dead", "20150724,dead" })); + + using var downloader = Downloader(); + var security = SecurityIdentifier.GenerateEquity(new DateTime(1980, 12, 12), "DEAD", Market.USA); + + Assert.AreEqual("DEAD", downloader.ResolveTicker(security, new DateTime(2015, 1, 5))?.ToUpperInvariant(), + "listed then"); + Assert.IsNull(downloader.ResolveTicker(security, new DateTime(2016, 2, 14)), "delisted by then"); + } + + [TestCase("20240201", "20230930")] // inside the December quarter's filing window + [TestCase("20240214", "20230930")] // the deadline itself + [TestCase("20240215", "20231231")] // the deadline has passed: September is stale + [TestCase("20240331", "20231231")] // a quarter end is not yet the new quarter + [TestCase("20240401", "20231231")] // the March quarter starts filing + [TestCase("20240516", "20240331")] + public void AQuarterStaysLiveUntilTheNextQuartersDeadlinePasses(string day, string oldest) + { + // Between a quarter end and its 45 day deadline both the finished quarter and the one + // filling in are live, which is the two-quarter rule. Once the deadline passes the older + // one is stale for everyone who files on time. + Assert.AreEqual(DateTime.ParseExact(oldest, "yyyyMMdd", null), + SEC13FDownloader.OldestLivePeriod(DateTime.ParseExact(day, "yyyyMMdd", null))); + } + + [Test] + public void ASecurityNobodyReportsAnyMoreLeavesTheUniverse() + { + // The forward fill used to carry a security's last row forever: 252 of the 6,886 + // securities in the May 2026 universe had nothing newer than 2024, some back to 2013, + // and Monsanto was still there with its March 2022 quarter. A quarter now leaves once + // the next one's deadline has passed without it being refreshed, and a security leaves + // after its delisting date. + SeedMapFileRows( + ("aapl", new[] { "19801212,aapl", "20501231,aapl" }), + ("stale", new[] { "19801212,stale", "20501231,stale" }), + ("gone", new[] { "19801212,gone", "20240305,gone" })); + + var holdings = Path.Combine(_root, "out", SEC13F.ReportFolder); + Directory.CreateDirectory(holdings); + File.WriteAllLines(Path.Combine(holdings, "aapl.csv"), new[] + { + "20240201 17:30,1,20231231,5,10,100,0,0,0,0,0,0", + "20240415 17:30,1,20240331,6,12,120,0,0,0,0,0,0", + "20240520 17:30,1,20240331,7,14,140,0,0,0,0,0,0" + }); + File.WriteAllLines(Path.Combine(holdings, "stale.csv"), new[] { "20240201 17:30,1,20231231,1,1,1,0,0,0,0,0,0" }); + File.WriteAllLines(Path.Combine(holdings, "gone.csv"), new[] { "20240202 17:30,1,20231231,2,2,2,0,0,0,0,0,0" }); + + using (var downloader = Downloader()) + { + downloader.BuildUniverseFiles(); + } + + var universe = Path.Combine(holdings, "universe"); + string[] Tickers(string day) => File.ReadAllLines(Path.Combine(universe, $"{day}.csv")) + .Where(line => line.Length > 0) + .Select(line => line.Split(',')[1]) + .OrderBy(ticker => ticker, StringComparer.Ordinal) + .ToArray(); + + Assert.AreEqual(new[] { "AAPL", "GONE", "STALE" }, Tickers("20240205"), "all three are live in February"); + Assert.AreEqual(new[] { "AAPL", "STALE" }, Tickers("20240306"), "GONE leaves after its delisting date"); + Assert.AreEqual(new[] { "AAPL", "STALE" }, Tickers("20240515"), + "the December quarter is still live up to the March quarter's deadline"); + Assert.AreEqual(new[] { "AAPL" }, Tickers("20240517"), + "STALE never reported March and leaves once that deadline has passed"); + } + + /// + /// Points LEAN's data folder at a map file archive holding exactly the rows given per ticker, + /// so a test can shape a listing, a rename or a delisting. See SeedMapFiles for why the data + /// folder and lookup date are pinned. + /// + private void SeedMapFileRows(params (string Ticker, string[] Rows)[] files) + { + var dataFolder = Path.Combine(_root, "data"); + var mapFiles = Path.Combine(dataFolder, "equity", "usa", "map_files"); + Directory.CreateDirectory(mapFiles); + + var lookupDate = new DateTime(2026, 1, 2); + using (var zip = ZipFile.Open( + Path.Combine(mapFiles, $"map_files_{lookupDate:yyyyMMdd}.zip"), ZipArchiveMode.Create)) + { + foreach (var (ticker, rows) in files) + { + using var entry = new StreamWriter(zip.CreateEntry($"{ticker}.csv").Open()); + foreach (var row in rows) + { + entry.WriteLine(row); + } + } + } + + // Null when absent, so the teardown can tell a missing key from an empty one. + if (!_configSeeded) + { + _previousDataFolder = Config.Get("data-folder", null); + _previousLookupDate = Config.Get("map-file-provider-lookup-date", null); + _configSeeded = true; + } + + Config.Set("data-folder", dataFolder); + Config.Set("map-file-provider-lookup-date", $"{lookupDate:yyyyMMdd}"); + Globals.Reset(); + } + + // ---- N-PORT ticker normalisation ---------------------------------------------------------- + + [TestCase("GOOGL US", "GOOGL")] // the Bloomberg style venue suffix + [TestCase("googl", "GOOGL")] + [TestCase("GOOGL", "GOOGL")] + [TestCase(" brk.b ", "BRK.B")] // share classes survive, they are the LEAN spelling + public void ANPortTickerIsNormalisedBeforeItIsVotedOn(string raw, string expected) + { + // IDENTIFIER_TICKER is free text written by fund administrators, so the same security + // arrives as "GOOGL", "GOOGL US" and "goog" across funds. It is normalised and then + // voted on across every fund that reported the security, rather than trusted row by row. + Assert.AreEqual(expected, SEC13FTickerCrosswalk.Normalize(raw)); + } + + [TestCase("N/A")] + [TestCase("")] + [TestCase(" ")] + [TestCase("912828YV6")] // a CUSIP typed into the ticker column + [TestCase("NOTATICKERATALL")] // longer than any US equity ticker + public void ANPortTickerThatIsNotOneIsRejected(string raw) + { + // The crosswalk feeds a map file lookup, and a ticker the map files do not know still + // produces a plausible looking Symbol with no data behind it, so anything that is not + // ticker-shaped is dropped here rather than downstream. + Assert.IsNull(SEC13FTickerCrosswalk.Normalize(raw)); + } + + [Test] + public void ACrosswalkTickerNamesTheSecurityItNamedWhenTheFundsReportedIt() + { + // Facebook's CUSIP reaches the crosswalk as META, the ticker funds report today. Resolved + // at each filing date instead, META named the Roundhill Metaverse ETF from 2021-06-30 to + // 2022-01-28, so seven months of Facebook's holders were published as that ETF's, and + // before it META named nothing at all. + SeedMapFileRows( + ("meta", new[] { "20120518,fb", "20220608,fb", "20501231,meta" }), + ("metv", new[] { "20210630,meta", "20220128,meta", "20501231,metv" })); + + using var downloader = Downloader(); + downloader.TickerCrosswalk = new Dictionary + { + ["30303M102"] = new("META", new DateTime(2026, 1, 1)) + }; + + var security = downloader.ResolveThroughTicker("30303M102"); + + Assert.AreEqual("FB", downloader.ResolveTicker(security, new DateTime(2021, 10, 15))?.ToUpperInvariant(), + "Facebook's filing lands in Facebook's file, not the ETF's"); + Assert.AreEqual("FB", downloader.ResolveTicker(security, new DateTime(2016, 2, 12))?.ToUpperInvariant(), + "and the years before the ETF existed resolve too"); + Assert.AreEqual("META", downloader.ResolveTicker(security, new DateTime(2023, 2, 14))?.ToUpperInvariant()); + } + + [Test] + public void ACrosswalkTickerThatNamedNoUsSecurityWhenObservedResolvesToNothing() + { + // Barrick's CUSIP reached the crosswalk as ABX, its Toronto ticker, in data reported from + // July 2025. No US security traded as ABX then, and the resolver still returned the + // company that took the ticker in December, so its holders would have gone there. + SeedMapFileRows( + ("abx", new[] { "20200914,eres", "20230703,eres", "20251229,abl", "20501231,abx" }), + ("b", new[] { "19980102,abx", "20181231,abx", "20250508,gold", "20501231,b" })); + + using var downloader = Downloader(); + downloader.TickerCrosswalk = new Dictionary + { + ["067901108"] = new("ABX", new DateTime(2025, 7, 1)) + }; + + Assert.IsNull(downloader.ResolveThroughTicker("067901108")); + } + + // ---- security database rows repeated across lineages ---------------------------------------- + + [Test] + public void ACusipOnSeveralDatabaseRowsResolvesToTheRowCarryingTheIsinItBuilds() + { + // The security database repeats Alcoa's CUSIP on the old Alcoa, which trades as Howmet today and + // carries Howmet's ISIN, and on the Alcoa spun off in 2016. LEAN's resolver takes the first row, so + // the new Alcoa's holders went to Howmet's lineage. Both rows trade on the date; the one carrying the + // ISIN the CUSIP builds is the security the CUSIP names. + var oldAlcoa = SecurityIdentifier.GenerateEquity(new DateTime(1998, 1, 2), "AA", Market.USA); + var newAlcoa = SecurityIdentifier.GenerateEquity(new DateTime(2016, 11, 1), "AA", Market.USA); + SeedMapFileRows( + ("hwm", new[] { "19980102,aa", "20161031,aa", "20200331,arnc", "20501231,hwm" }), + ("aa", new[] { "20161101,aa", "20501231,aa" })); + SeedSecurityDatabase( + $"{oldAlcoa},01387210,,,US4432011082,4281", + $"{newAlcoa},01387210,,,US0138721065,1675149"); + + using var downloader = Downloader(); + + Assert.AreEqual(newAlcoa, downloader.ResolveSecurity("013872106", new DateTime(2022, 11, 14))); + } + + [Test] + public void ACusipWhoseFirstDatabaseRowNoLongerTradesResolvesToTheRowThatDoes() + { + // TG Therapeutics' CUSIP and ISIN sit on its current listing and on the one it had as Atlantic + // Technology Ventures, whose map file ends in 2005. The first row names a security with no ticker + // since, so every TG holder was dropped as belonging to a ticker another security owned. + var atlantic = SecurityIdentifier.GenerateEquity(new DateTime(1998, 1, 2), "ATLC", Market.USA); + var tg = SecurityIdentifier.GenerateEquity(new DateTime(2012, 10, 1), "TGTX", Market.USA); + SeedMapFileRows( + ("mhan", new[] { "19980102,atlc", "20040630,atlc", "20051230,mhan" }), + ("tgtx", new[] { "20121001,tgtx", "20501231,tgtx" })); + SeedSecurityDatabase( + $"{atlantic},88322Q10,,,US88322Q1085,", + $"{tg},88322Q10,,,US88322Q1085,1001316"); + + using var downloader = Downloader(); + + Assert.AreEqual(tg, downloader.ResolveSecurity("88322Q108", new DateTime(2025, 2, 14))); + } + + [Test] + public void ACusipWhoseDatabaseRowsAllStoppedTradingFallsThroughToTheCrosswalk() + { + // A row that no longer trades used to end the search: its security was dropped later as the owner + // of no ticker, and the N-PORT crosswalk, which knew the CUSIP, was never asked. + var cusip = "12345678" + SEC13FDownloader.ComputeCusipCheckDigit("12345678"); + var dead = SecurityIdentifier.GenerateEquity(new DateTime(1998, 1, 2), "OLDCO", Market.USA); + var live = SecurityIdentifier.GenerateEquity(new DateTime(2015, 3, 2), "NEWCO", Market.USA); + SeedMapFileRows( + ("oldco", new[] { "19980102,oldco", "20101231,oldco" }), + ("newco", new[] { "20150302,newco", "20501231,newco" })); + SeedSecurityDatabase($"{dead},12345678,,,,"); + + using var downloader = Downloader(); + downloader.TickerCrosswalk = new Dictionary + { + [cusip] = new("NEWCO", new DateTime(2025, 7, 1)) + }; + + Assert.AreEqual(live, downloader.ResolveSecurity(cusip, new DateTime(2025, 11, 14))); + } + + /// + /// Writes the security database the downloader reads at construction into the seeded data folder, + /// rows as the real file has them: SID, CUSIP without its check digit, FIGI, SEDOL, ISIN and CIK. + /// + private void SeedSecurityDatabase(params string[] rows) + { + var folder = Path.Combine(_root, "data", "symbol-properties"); + Directory.CreateDirectory(folder); + File.WriteAllLines(Path.Combine(folder, "security-database.csv"), rows); + } + + [TestCase("BRK.B", true)] + [TestCase("AAPL", true)] + [TestCase("UA.C ", false)] // the map files carry a trailing space for Under Armour's class C + [TestCase("A/B", false)] + [TestCase("X|Y", false)] + public void OnlyATickerLeanCanAskForBecomesAFile(string ticker, bool expected) + { + // A Symbol cannot hold a space or a '|', so a file named after such a ticker is one LEAN + // never reads, and every row in it failed when the delivery archive was read back. + Assert.AreEqual(expected, SEC13FDownloader.IsFileNameSafe(ticker)); + } + + [Test] + public void ASecurityHasNoTickerBeforeItBeganTrading() + { + // Before its first row a map file answers with its first ticker, so a filing dated + // before the listing would land under a ticker the security did not have yet. + SeedMapFileRows(("late", new[] { "20200914,late", "20501231,late" })); + + using var downloader = Downloader(); + var security = SecurityIdentifier.GenerateEquity(new DateTime(2020, 9, 14), "LATE", Market.USA); + + Assert.IsNull(downloader.ResolveTicker(security, new DateTime(2019, 6, 3)), "not listed yet"); + Assert.AreEqual("LATE", downloader.ResolveTicker(security, new DateTime(2021, 1, 4))?.ToUpperInvariant()); + } + + [Test] + public void TheNewestObservationOfACusipWinsWhateverOrderTheQuartersArrive() + { + // The quarters used to be folded newest first, so a first build kept the oldest ticker + // of a renamed security while a refresh kept the newest. + var older = new[] { KeyValuePair.Create("30303M102", "FB") }; + var newer = new[] { KeyValuePair.Create("30303M102", "META") }; + + var olderFirst = new Dictionary(); + SEC13FTickerCrosswalk.Fold(olderFirst, new DateTime(2022, 4, 1), older); + SEC13FTickerCrosswalk.Fold(olderFirst, new DateTime(2022, 7, 1), newer); + + var newerFirst = new Dictionary(); + SEC13FTickerCrosswalk.Fold(newerFirst, new DateTime(2022, 7, 1), newer); + SEC13FTickerCrosswalk.Fold(newerFirst, new DateTime(2022, 4, 1), older); + + Assert.AreEqual("META", olderFirst["30303M102"].Ticker); + Assert.AreEqual("META", newerFirst["30303M102"].Ticker); + } + + // ---- Crosswalk resolutions are held against the close -------------------------------------- + + [Test] + public void ACrosswalkGroupStaysUnlessItsPricesSayItIsAnotherSecurity() + { + // The crosswalk is a fund administrator's ticker, so a CUSIP can reach the wrong company. + // The group's own prices against the quarter-end close decide, on the day it is read. + SeedCloses("20251231", ("etsy", 120m), ("defi", 129.50m), ("aapl", 250m), ("spy", 24.23m)); + using var downloader = Downloader(); + var period = new DateTime(2025, 12, 31); + var filed = new DateTime(2026, 2, 14); + bool Keeps(string cusip, string ticker, params double[] prices) => + downloader.KeepsCrosswalkGroup(cusip, ListedSince1980(ticker), period, filed, prices.ToList()); + + // Etsy's convertible notes reached Etsy's stock: 171 million of its 298 million shares. + Assert.IsFalse(Keeps("29786AAJ5", "ETSY"), "a note reported as principal carries no price"); + Assert.IsFalse(Keeps("29786AAJ5", "ETSY", 1.02, 0.98), "a note reported as shares trades near par"); + Assert.IsTrue(Keeps("46435GAA0", "SPY", 24.23, 24.20), + "a CUSIP with letters that trades at the security's own price, as the iBonds ETFs do, stays"); + + // DeFi Technologies at $2 reached the $129.50 Hashdex DEFI ETF through its ticker. + Assert.IsFalse(Keeps("244916102", "DEFI", 2.11, 2.05, 2.11), "three managers at $2 are another company"); + Assert.IsTrue(Keeps("037833100", "AAPL", 25.0), "one manager's odd price is a slip, not another company"); + Assert.IsTrue(Keeps("037833100", "AAPL", 0.25, 0.25, 250.1), "the same prices in thousands match"); + Assert.IsTrue(Keeps("037833100", "AAPL"), "a day of option lines only cannot be checked and stays"); + Assert.IsTrue(Keeps("594918104", "MSFT", 2.0, 2.0, 2.0), "nor can a security the close file does not carry"); + + // Avanos on 5 February 2026: four managers in thousands and four in dollars. Their median + // sat half way between the two units and dropped the group. + Assert.IsTrue(Keeps("037833100", "AAPL", 0.25, 0.25, 0.25, 0.25, 250.0, 250.1, 249.9, 740.8), + "a day that mixes the two units is still the security"); + } + + [Test] + public void ALineThatSitsOnNoUnitStepIsNotScaled() + { + // A price about a hundred times below the close is neither dollars nor thousands: a bond at + // par against its issuer's stock, or another company. Scaling it by the nearest step put + // Seagate's December 2025 quarter at $413 billion. + SeedCloses("20251231", ("aapl", 250m)); + var filing = new DateTime(2026, 2, 14); + var quarter = new DateTime(2025, 12, 31); + var holdings = ReadInfoTable( + new[] + { + Line("0000000000-00-000001", "037833100", "250000", "1000", "SH"), + Line("0000000000-00-000002", "037833100", "2500", "1000", "SH") + }, + resolve: true, + Submission("0000000000-00-000001", filing, quarter, cik: 1), + Submission("0000000000-00-000002", filing, quarter, cik: 2)); + + Assert.AreEqual(250000m + 2500m, holdings.Values.Single().HoldingValue, + "the $2.50 line keeps its filing's dollar factor instead of becoming $2.5 million"); + } + + [Test] + public void ABrokenFilingIsNeverScaledUp() + { + // Every line reads $1,000 a share, since SSHPRNAMT carries VALUE in thousands. Two of the + // four prices land on a unit step by chance, not most, so the filing shows no unit. Before + // 2023 the rule multiplied it by a thousand, and one such manager put Alphabet's September + // 2020 quarter 9.5 percent above its close. + SeedCloses("20200930", ("aapl", 100m), ("msft", 200m), ("nvda", 500m), ("spy", 330m)); + var holdings = ReadInfoTable( + new[] + { + Line("0000000000-00-000001", "037833100", "1000000", "1000", "SH"), + Line("0000000000-00-000001", "594918104", "1000000", "1000", "SH"), + Line("0000000000-00-000001", "67066G104", "1000000", "1000", "SH"), + Line("0000000000-00-000001", "78462F103", "1000000", "1000", "SH") + }, + resolve: true, + Submission("0000000000-00-000001", new DateTime(2020, 11, 6), new DateTime(2020, 9, 30), cik: 1)); + + Assert.AreEqual(4, holdings.Count); + Assert.IsTrue(holdings.Values.All(holding => holding.HoldingValue == 1000000m), + string.Join(", ", holdings.Values.Select(holding => holding.HoldingValue))); + } + + [TestCase("20200814", "20200630", "20200630", "250000", "2500", 752500)] // dollars before 2023: the rule made the $2.50 line $2,500 + [TestCase("20230214", "20221231", "20221230", "250", "25", 750025)] // thousands in 2023: the filing's unit made it 1,000 times larger + public void ALineOnNoStepIsNeverScaledPastItsFilingOrItsDate(string filed, string quarter, string closeDay, + string onStep, string offStep, decimal expected) + { + // Three lines prove the filing's unit and a fourth sits on no step. It cannot be checked, so + // it takes the smaller of the filing's unit and the rule of the filing date: either one alone + // inflated one era by hundreds of billions. + SeedCloses(closeDay, ("aapl", 250m)); + var holdings = ReadInfoTable( + new[] + { + Line("0000000000-00-000001", "037833100", onStep, "1000", "SH"), + Line("0000000000-00-000001", "037833100", onStep, "1000", "SH"), + Line("0000000000-00-000001", "037833100", onStep, "1000", "SH"), + Line("0000000000-00-000001", "037833100", offStep, "1000", "SH") + }, + resolve: true, + Submission("0000000000-00-000001", Time.ParseDate(filed), Time.ParseDate(quarter), cik: 1)); + + Assert.AreEqual(expected, holdings.Values.Single().HoldingValue); + } + + [TestCase(0.0, 0)] + [TestCase(-3.0, -1)] + [TestCase(3.1, 1)] + [TestCase(-2.9, -1)] + public void AnOffsetNearAThousandfoldStepIsAUnit(double offset, int step) + { + Assert.AreEqual(step, SEC13FDownloader.UnitStep(offset)); + } + + [TestCase(-1.5)] // half way between dollars and thousands + [TestCase(-2.0)] // a hundred times below: a bond at par against a stock + [TestCase(1.79)] + [TestCase(-6.0)] // a millionfold step is a share count slip, not a unit + public void AnOffsetOnNoStepIsNoUnit(double offset) + { + Assert.IsNull(SEC13FDownloader.UnitStep(offset)); + } + + [Test] + public void TheCloseIsNeverReadAfterTheFilingDate() + { + // A period typed ahead of its filing date would otherwise read a price nobody had when + // the filing was made. The quarter ends on a Tuesday; filed the Sunday before, the + // Friday's close is the newest one there was. + SeedCloses("20260327", ("aapl", 250m)); + var coarse = Path.Combine(_root, "data", "equity", "usa", "fundamental", "coarse"); + File.WriteAllText(Path.Combine(coarse, "20260331.csv"), $"{ListedSince1980("AAPL")},AAPL,999,1,1,True,1,1\n"); + var prices = new SEC13FClosePrices(coarse); + + Assert.AreEqual(250m, prices.Close(ListedSince1980("AAPL"), new DateTime(2026, 3, 31), new DateTime(2026, 3, 29))); + Assert.AreEqual(999m, prices.Close(ListedSince1980("AAPL"), new DateTime(2026, 3, 31), new DateTime(2026, 4, 2))); + } + + // ---- Reading a day from EDGAR --------------------------------------------------------------- + + [Test] + public void TheDailyIndexYieldsTheHoldingsReportsAndTheirAmendmentsOnly() + { + // Notices carry no information table and the data sets' reader skips them too. + var index = string.Join("\n", + "Form Type Company Name CIK Date Filed File Name", + "---------------------------------------------------------", + "10-K SOMETHING ELSE INC 1111111 20260814 edgar/data/1111111/0001111111-26-000001.txt", + "13F-HR &PARTNERS 107136 20260814 edgar/data/107136/0001214659-26-010148.txt", + "13F-HR/A FUND 1 ADVISERS LLC 2222222 20260814 edgar/data/2222222/0002222222-26-000003.txt", + "13F-NT NOTICE FILER LP 3333333 20260814 edgar/data/3333333/0003333333-26-000004.txt"); + + var entries = SEC13FEdgarDay.ParseIndex(index); + + Assert.AreEqual(new[] { "0001214659-26-010148", "0002222222-26-000003" }, + entries.Select(entry => entry.Accession).ToArray()); + Assert.AreEqual(new[] { "13F-HR", "13F-HR/A" }, entries.Select(entry => entry.FormType).ToArray()); + Assert.AreEqual(new[] { 107136, 2222222 }, entries.Select(entry => entry.Cik).ToArray()); + Assert.AreEqual(new DateTime(2026, 8, 14), entries[0].Filed); + } + + [Test] + public void AFilingIsReadFromItsSubmissionFile() + { + // The period and the confidential flag come from the primary document, the lines from the + // information table, whatever namespace the filer's software wrote. + var filing = SEC13FEdgarDay.ParseFiling(SampleEntry(), SampleSubmission()); + + Assert.AreEqual("0001214659-26-010148", filing.Accession); + Assert.AreEqual(new DateTime(2026, 6, 30), filing.Period); + Assert.IsTrue(filing.ConfidentialOmitted); + Assert.AreEqual(2, filing.Lines.Count); + Assert.AreEqual(new[] { "88025U109", "274974", "7172", "SH", null, "7172", "0" }, filing.Lines[0]); + Assert.AreEqual("Put", filing.Lines[1][4]); + } + + [Test] + public void ADaysArchiveCarriesTheTablesTheProcessorReads() + { + // The day is written in the data sets' own layout so the rest of the processor reads it + // like a window, which is what makes EDGAR and the data sets give the same rows. + var filing = SEC13FEdgarDay.ParseFiling(SampleEntry(), SampleSubmission()); + + using var buffer = new MemoryStream(); + SEC13FEdgarDay.WriteArchive(buffer, new[] { filing }); + buffer.Position = 0; + using var zip = new ZipArchive(buffer, ZipArchiveMode.Read); + + var submission = SEC13FFiles.ReadTable(zip, "edgar", "SUBMISSION.tsv", false, + "ACCESSION_NUMBER", "FILING_DATE", "SUBMISSIONTYPE", "CIK", "PERIODOFREPORT").Single(); + Assert.AreEqual("2026-08-14", submission.Fields[submission.Columns["FILING_DATE"]]); + Assert.AreEqual("2026-06-30", submission.Fields[submission.Columns["PERIODOFREPORT"]]); + + var summary = SEC13FFiles.ReadTable(zip, "edgar", "SUMMARYPAGE.tsv", false, + "ACCESSION_NUMBER", "ISCONFIDENTIALOMITTED").Single(); + Assert.AreEqual("Y", summary.Fields[summary.Columns["ISCONFIDENTIALOMITTED"]]); + + var lines = SEC13FFiles.ReadTable(zip, "edgar", "INFOTABLE.tsv", false, + "ACCESSION_NUMBER", "CUSIP", "VALUE", "SSHPRNAMT", "SSHPRNAMTTYPE", "PUTCALL", + "VOTING_AUTH_SOLE", "VOTING_AUTH_SHARED").ToList(); + Assert.AreEqual(2, lines.Count); + Assert.AreEqual("037833100", lines[1].Fields[lines[1].Columns["CUSIP"]]); + Assert.AreEqual("Put", lines[1].Fields[lines[1].Columns["PUTCALL"]]); + } + + [Test] + public void AnEdgarDayIsPublishedOnlyWhenItsQuarterListsItsIndex() + { + // EDGAR answers 403 both for an index that does not exist and for a reader it has blocked, + // so whether a day is out comes from its listings. A folder EDGAR has not created yet + // answers 403 too, so it is looked up in its parent first: requesting any folder missing + // from this map throws, and fails the test. + const string root = "https://www.sec.gov/Archives/edgar/daily-index/"; + var listings = new Dictionary> + { + [root] = new HashSet { "2025", "2026" }, + [root + "2026/"] = new HashSet { "QTR1", "QTR2", "QTR3" }, + [root + "2026/QTR3/"] = new HashSet { "form.20260904.idx", "form.20260908.idx" } + }; + + Assert.IsTrue(SEC13FEdgarDay.IsIndexPublished(new DateTime(2026, 9, 8), url => listings[url])); + Assert.IsFalse(SEC13FEdgarDay.IsIndexPublished(new DateTime(2026, 9, 7), url => listings[url]), "Labor Day"); + Assert.IsFalse(SEC13FEdgarDay.IsIndexPublished(new DateTime(2026, 10, 1), url => listings[url]), "a new quarter"); + Assert.IsFalse(SEC13FEdgarDay.IsIndexPublished(new DateTime(2027, 1, 4), url => listings[url]), "a new year"); + } + + [Test] + public void AnEdgarBlockFailsTheDayInsteadOfSkippingIt() + { + // Taken for "no index", a blocked day was never recorded and fell out of reach of the + // daily run's lookback, so the rebuild lost it for good. + HttpRequestException Blocked() => new("Forbidden", null, HttpStatusCode.Forbidden); + + Assert.Throws(() => SEC13FEdgarDay.Build(new DateTime(2026, 9, 8), _root, + _ => throw Blocked(), _ => throw Blocked())); + } + + [TestCase(HttpStatusCode.NotFound, true)] + [TestCase(HttpStatusCode.ServiceUnavailable, true)] + [TestCase(HttpStatusCode.TooManyRequests, true)] + [TestCase(HttpStatusCode.Forbidden, false)] + public void OnlyABlockIsNotAskedAgain(HttpStatusCode status, bool retried) + { + // Every file requested is one the SEC lists, so a 404 is a hiccup worth asking again: a + // filing in the 24 July 2026 index answered 404 during a rebuild and 200 afterwards. + Assert.AreEqual(retried, SEC13FDownloader.IsWorthRetrying(new HttpRequestException("", null, status))); + } + + [Test] + public void AnEdgarListingYieldsItsNames() + { + const string listing = @"{""directory"":{""item"":[{""last-modified"":""09\/08\/2026 10:02:29 PM""," + + @"""name"":""form.20260908.idx"",""type"":""file"",""href"":""form.20260908.idx"",""size"":""782 KB""}]," + + @"""name"":""daily-index\/2026\/QTR3\/"",""parent-dir"":""..\/""}}"; + + Assert.AreEqual(new[] { "form.20260908.idx" }, SEC13FEdgarDay.ListingNames(listing).ToArray()); + Assert.Throws(() => SEC13FEdgarDay.ListingNames("{}")); + } + + private static SEC13FEdgarDay.IndexEntry SampleEntry() + { + return new SEC13FEdgarDay.IndexEntry("13F-HR", 107136, new DateTime(2026, 8, 14), + "edgar/data/107136/0001214659-26-010148.txt"); + } + + /// A full submission file cut down to the two documents the reader uses. + private static string SampleSubmission() + { + return string.Join("\n", + "0001214659-26-010148.txt : 20260814", + "", + "13F-HR", + "", + "", + "", + "" + + "13F-HR06-30-2026" + + "true" + + "", + "", + "", + "", + "", + "INFORMATION TABLE", + "", + "", + "" + + "88025U109274974" + + "7172SH" + + "717200" + + "" + + "0378331001000" + + "50SH" + + "Put050" + + "0", + "", + "", + "", + ""); + } + + // ---- Fixtures ----------------------------------------------------------------------------- + + /// One increment or cumulative row. Only the first two measures are exercised. + private static SEC13FDownloader.HoldingsRow Row(string time, string periodEnd, decimal holders, + decimal shares, bool confidential = false) + { + return new SEC13FDownloader.HoldingsRow + { + Time = DateTime.ParseExact(time, "yyyyMMdd HH:mm", null), + PeriodEnd = DateTime.ParseExact(periodEnd, "yyyyMMdd", null), + Values = new[] { holders, shares, 0m, 0m, 0m, 0m, 0m, 0m }, + ConfidentialOmitted = confidential + }; + } + + /// One INFOTABLE.tsv line, in the column order the header below declares. + private static string Line(string accession, string cusip, string value, string amount, string shareType, + string putCall = "", string votingSole = "0", string votingShared = "0") + { + return string.Join('\t', accession, cusip, value, amount, shareType, putCall, votingSole, votingShared); + } + + /// One SUBMISSION row, keyed by its accession number the way the processor keys it. + private static KeyValuePair Submission(string accession, + DateTime filingDate, DateTime period, int cik = 1, bool isAmendment = false) + { + return new KeyValuePair(accession, + new SEC13FDownloader.Submission + { + Cik = cik, + FilingDate = filingDate, + Period = period, + IsAmendment = isAmendment + }); + } + + /// + /// Runs the real INFOTABLE reader over an in-memory archive. The unit break and the amendment + /// rule live inside that loop, so a test that rebuilt them here would only be asserting + /// itself; this drives the shipped code over the table shape the SEC publishes. + /// + private Dictionary ReadInfoTable( + string[] lines, params KeyValuePair[] submissions) + { + return ReadInfoTable(lines, false, submissions); + } + + /// + /// The same reader, with the four CUSIPs the unit tests use resolvable through the crosswalk + /// when is set, so their lines are held against the closes + /// SeedCloses wrote. Unset, no CUSIP resolves and every line falls back to its filing's rule. + /// + private Dictionary ReadInfoTable( + string[] lines, bool resolve, params KeyValuePair[] submissions) + { + var text = new StringBuilder() + .AppendLine(string.Join('\t', "ACCESSION_NUMBER", "CUSIP", "VALUE", "SSHPRNAMT", + "SSHPRNAMTTYPE", "PUTCALL", "VOTING_AUTH_SOLE", "VOTING_AUTH_SHARED")); + + foreach (var line in lines) + { + text.AppendLine(line); + } + + using var buffer = new MemoryStream(); + using (var writing = new ZipArchive(buffer, ZipArchiveMode.Create, true)) + using (var entry = new StreamWriter(writing.CreateEntry("INFOTABLE.tsv").Open())) + { + entry.Write(text.ToString()); + } + + buffer.Position = 0; + using var archive = new ZipArchive(buffer, ZipArchiveMode.Read); + using var downloader = Downloader(); + downloader.TickerCrosswalk = resolve + ? UnitTestCrosswalk() + : new Dictionary(); + + return downloader.ReadInfoTable( + archive, + new SEC13FDownloader.Archive("2024q1_form13f.zip", "https://localhost/2024q1_form13f.zip", + new DateTime(2024, 1, 1), new DateTime(2024, 3, 31)), + submissions.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal)); + } + + /// The CUSIPs the unit tests name, to the tickers SeedCloses lists. + private static Dictionary UnitTestCrosswalk() + { + var observed = new DateTime(2026, 1, 1); + return new Dictionary + { + ["037833100"] = new("AAPL", observed), + ["594918104"] = new("MSFT", observed), + ["67066G104"] = new("NVDA", observed), + ["78462F103"] = new("SPY", observed) + }; + } + + /// + /// Writes one trading day's coarse file with these closes, and map files listing the tickers + /// over every date used, so a crosswalk CUSIP resolves to the identifier the file is keyed by. + /// + private void SeedCloses(string day, params (string Ticker, decimal Close)[] closes) + { + SeedMapFiles("aapl", "msft", "nvda", "spy", "defi", "etsy"); + + var coarse = Path.Combine(_root, "data", "equity", "usa", "fundamental", "coarse"); + Directory.CreateDirectory(coarse); + File.WriteAllLines(Path.Combine(coarse, $"{day}.csv"), closes.Select(pair => + $"{ListedSince1980(pair.Ticker)},{pair.Ticker.ToUpperInvariant()}," + + $"{pair.Close.ToString(System.Globalization.CultureInfo.InvariantCulture)},100,1000,True,1,1")); + } + + /// The identifier SeedMapFiles gives a ticker: listed on its first row, 1980-12-12. + private static SecurityIdentifier ListedSince1980(string ticker) + { + return SecurityIdentifier.GenerateEquity(new DateTime(1980, 12, 12), ticker.ToUpperInvariant(), Market.USA); + } + + /// + /// Runs the real universe build over a single per-security file and reads back which quarters + /// each business day published, keyed by the day. The retirement rule is what is being + /// asserted, so the whole forward-fill is driven rather than the rule on its own. + /// + private Dictionary BuildUniverse(params string[] rows) + { + SeedMapFiles("aapl"); + + var destination = Path.Combine(_root, "out"); + var holdings = Path.Combine(destination, SEC13F.ReportFolder); + Directory.CreateDirectory(holdings); + File.WriteAllLines(Path.Combine(holdings, "aapl.csv"), rows); + + using (var downloader = Downloader()) + { + downloader.BuildUniverseFiles(); + } + + var universe = Path.Combine(holdings, "universe"); + Assert.IsTrue(Directory.Exists(universe), $"no universe written at {universe}"); + + return Directory.GetFiles(universe, "*.csv").ToDictionary( + Path.GetFileNameWithoutExtension, + file => File.ReadAllLines(file) + .Where(line => line.Length > 0) + .SelectMany(QuartersOf) + .OrderBy(period => period, StringComparer.Ordinal) + .ToArray()); + } + + /// + /// The quarters one universe line carries: the identifier and the ticker, then how many + /// quarters follow and that many groups, each of them starting with its reported quarter. + /// The group width is read from the line rather than assumed, the way the data type reads it. + /// + private static IEnumerable QuartersOf(string line) + { + const int identifierColumns = 3; + var csv = line.Split(','); + var quarters = int.Parse(csv[2], System.Globalization.CultureInfo.InvariantCulture); + var width = (csv.Length - identifierColumns) / quarters; + + return Enumerable.Range(0, quarters).Select(quarter => csv[identifierColumns + quarter * width]); + } + + /// + /// Points LEAN's data folder at a map file archive this test wrote, holding one row per + /// ticker that spans every date used here. + /// + /// The universe build turns each ticker into a SecurityIdentifier point in time, and + /// LocalZipMapFileProvider throws outright when it finds no archive at all, so without this + /// the test could only be skipped on a machine with no LEAN data checkout, which is most of + /// them. The lookup date is pinned rather than left to walk back from yesterday, so the + /// fixture does not expire. + /// + private void SeedMapFiles(params string[] tickers) + { + SeedMapFileRows(tickers + .Select(ticker => (ticker, new[] { $"19801212,{ticker}", $"20501231,{ticker}" })) + .ToArray()); + } + + /// A downloader writing into this test's own temporary directories. + private SEC13FDownloader Downloader() + { + return new SEC13FDownloader(Path.Combine(_root, "out"), Path.Combine(_root, "processed"), null); + } + } +} diff --git a/tests/SEC13FTests.cs b/tests/SEC13FTests.cs new file mode 100644 index 0000000..43d616d --- /dev/null +++ b/tests/SEC13FTests.cs @@ -0,0 +1,735 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using NUnit.Framework; +using ProtoBuf; +using QuantConnect.Data; +using QuantConnect.DataSource; +using QuantConnect.Python; + +namespace QuantConnect.DataLibrary.Tests +{ + /// + /// Unit tests for and . Every case parses an + /// in-line sample line, so the fixture needs no data on disk and is safe to run in CI. The one + /// test that touches the processed output is skipped when that folder is not there yet. + /// + [TestFixture] + public class SEC13FTests + { + // Frozen line layout: the release stamp, how many quarters the release restated, and then + // that many groups of ten columns, oldest quarter first: + // PeriodEnd,Holders,Shares,HoldingValue,CallShares,PutShares,PrincipalValue,VotingSole,VotingShared,ConfidentialOmitted + private const string FullLine = + "20240214 03:00,1,20231231,4812,10589324611,3210456789012,152340000,98760000,4500000,9012345678,1234567890,1"; + + // Same quarter, every optional measure withheld. Must produce nulls, never zeros. + private const string EmptyValuesLine = "20240214 03:00,1,20231231,,,,,,,,,0"; + + // One release restating three quarters at once, which is what the layout exists for. The + // middle quarter is the one most managers have reported. + private const string ThreeQuarterLine = + "20240515 03:00,3," + + "20230630,11,110,1100,0,0,0,0,0,0," + + "20230930,4800,48000,480000,0,0,0,0,0,1," + + "20231231,12,120,1200,0,0,0,0,0,0"; + + private static SubscriptionDataConfig Config(string ticker = "AAPL") + { + return new SubscriptionDataConfig( + typeof(SEC13F), + Symbol.Create(ticker, SecurityType.Base, Market.USA), + Resolution.Daily, + TimeZones.NewYork, + TimeZones.NewYork, + false, false, false); + } + + private static SEC13F Read(string line, string ticker = "AAPL") + { + return new SEC13F().Reader(Config(ticker), line, DateTime.UtcNow, false) as SEC13F; + } + + [Test] + public void ReaderParsesEveryColumn() + { + var point = Read(FullLine); + + Assert.IsNotNull(point); + Assert.AreEqual(Symbol.Create("AAPL", SecurityType.Base, Market.USA), point.Symbol); + Assert.AreEqual(new DateTime(2024, 2, 14, 3, 0, 0), point.Time); + Assert.AreEqual(point.Time, point.EndTime); + Assert.AreEqual(1, point.Holdings.Count); + + var quarter = point.Holdings[0]; + Assert.AreEqual(new DateTime(2023, 12, 31), quarter.PeriodEnd); + Assert.AreEqual("2023Q4", quarter.Quarter); + Assert.AreEqual(4812m, quarter.Holders); + Assert.AreEqual(10589324611m, quarter.Shares); + Assert.AreEqual(3210456789012m, quarter.HoldingValue); + Assert.AreEqual(152340000m, quarter.CallShares); + Assert.AreEqual(98760000m, quarter.PutShares); + Assert.AreEqual(4500000m, quarter.PrincipalValue); + Assert.AreEqual(9012345678m, quarter.VotingSole); + Assert.AreEqual(1234567890m, quarter.VotingShared); + Assert.IsTrue(quarter.ConfidentialOmitted); + Assert.AreEqual(3210456789012m, point.Value); + } + + [Test] + public void EveryQuarterCarriesThePointsSymbol() + { + // A quarter reaches pandas as a row of its own, so it has to name the security it + // belongs to rather than leave the frame to guess from the row before it. + var point = Read(ThreeQuarterLine, "MSFT"); + + Assert.IsTrue(point.Holdings.All(holding => holding.Symbol == point.Symbol)); + } + + [Test] + public void OneReleaseCarriesEveryQuarterItRestated() + { + // The reason the layout changed. LEAN hands an algorithm a single data point per + // security per timestamp, so the quarters of one release shipped as separate lines + // reached the algorithm one at a time and the rest were dropped in silence. + var point = Read(ThreeQuarterLine); + + Assert.IsNotNull(point); + Assert.AreEqual(3, point.Holdings.Count); + Assert.AreEqual( + new[] { "2023Q2", "2023Q3", "2023Q4" }, + point.Holdings.Select(holding => holding.Quarter).ToArray(), + "the quarters have to come out oldest first, the order the file writes them in"); + Assert.AreEqual(new[] { 11m, 4800m, 12m }, point.Holdings.Select(holding => holding.Holders).ToArray()); + Assert.IsTrue(point.Holdings.All(holding => holding.PeriodEnd < point.Time), + "every quarter of a release ends before the release"); + } + + [Test] + public void EnumeratingThePointWalksItsQuarters() + { + // The IEnumerable is what lets pandas flatten one point into one row per quarter. + var point = Read(ThreeQuarterLine); + + Assert.AreEqual(point.Holdings, point.ToList()); + } + + [Test] + public void MostReportedIsTheQuarterMostManagersHaveReported() + { + // While a new quarter fills in, over roughly six weeks, the quarter before it still + // holds more managers and is the one that answers how widely a name is held. It is also + // the reading whose columns the DataFrame carries, and the one Value mirrors. + var point = Read(ThreeQuarterLine); + + Assert.AreEqual(new DateTime(2023, 9, 30), point.MostReported.PeriodEnd); + Assert.AreEqual(4800m, point.MostReported.Holders); + Assert.AreEqual(480000m, point.Value); + } + + [Test] + public void MostReportedPrefersTheNewerQuarterWhenTwoAreTied() + { + // Two quarters with the same number of managers is the handover finishing: the newer + // one is the current reading, so the tie goes to it rather than to file order. + var point = Read( + "20240515 03:00,2," + + "20230930,4800,48000,480000,0,0,0,0,0,0," + + "20231231,4800,49000,490000,0,0,0,0,0,0"); + + Assert.AreEqual(new DateTime(2023, 12, 31), point.MostReported.PeriodEnd); + Assert.AreEqual(490000m, point.Value); + } + + [Test] + public void LatestIsTheNewestQuarterOfTheRelease() + { + var point = Read(ThreeQuarterLine); + + Assert.AreEqual(new DateTime(2023, 12, 31), point.Latest.PeriodEnd); + Assert.AreSame(point.Holdings[point.Holdings.Count - 1], point.Latest); + } + + [Test] + public void LatestIsKeptOutOfTheDataFrame() + { + // Two expanded readings on one point would prefix every column with the name of the + // property it came from, so only MostReported is expanded. + Assert.IsNotNull( + typeof(SEC13F).GetProperty(nameof(SEC13F.Latest)).GetCustomAttribute(), + "Latest has to carry PandasIgnore"); + Assert.IsNull( + typeof(SEC13F).GetProperty(nameof(SEC13F.MostReported)).GetCustomAttribute(), + "MostReported is the reading the frame expands"); + } + + [Test] + public void CloneCopiesEveryProperty() + { + var original = Read(ThreeQuarterLine); + var clone = original.Clone() as SEC13F; + + Assert.IsNotNull(clone); + // Reflection rather than a hand-written list: a property added to the class but forgotten + // in Clone() is exactly the bug this test has to catch, and a hand-written list cannot. + AssertPropertiesAndFieldsAreEqual(original, clone); + } + + [Test] + public void CloneCopiesTheQuartersRatherThanSharingThem() + { + // The clone is what reaches the algorithm, which may keep it. Sharing the quarters + // would let the next release mutate a point that has already been read. + var original = Read(ThreeQuarterLine); + var clone = original.Clone() as SEC13F; + + Assert.AreNotSame(original.Holdings, clone.Holdings); + for (var i = 0; i < original.Holdings.Count; i++) + { + Assert.AreNotSame(original.Holdings[i], clone.Holdings[i]); + } + } + + [Test] + public void ValueMirrorsTheMostReportedHoldingValue() + { + Assert.AreEqual(Read(FullLine).MostReported.HoldingValue, Read(FullLine).Value); + // A withheld HoldingValue leaves the point at zero rather than throwing or carrying null. + Assert.AreEqual(0m, Read(EmptyValuesLine).Value); + } + + [Test] + public void EmptyNumericColumnsBecomeNullNotZero() + { + var quarter = Read(EmptyValuesLine)?.Holdings.SingleOrDefault(); + + Assert.IsNotNull(quarter); + Assert.IsNull(quarter.Holders); + Assert.IsNull(quarter.Shares); + Assert.IsNull(quarter.HoldingValue); + Assert.IsNull(quarter.CallShares); + Assert.IsNull(quarter.PutShares); + Assert.IsNull(quarter.PrincipalValue); + Assert.IsNull(quarter.VotingSole); + Assert.IsNull(quarter.VotingShared); + } + + [TestCase("1", true)] + [TestCase("0", false)] + [TestCase("", false)] + public void ConfidentialOmittedParsesFlag(string flag, bool expected) + { + var line = $"20240214 03:00,1,20231231,4812,10589324611,3210456789012,,,,,,{flag}"; + + Assert.AreEqual(expected, Read(line).Holdings.Single().ConfidentialOmitted); + } + + [Test] + public void TheConfidentialFlagBelongsToTheQuarterNotTheRelease() + { + // One release restates several quarters and only some of them may be missing positions + // withheld under confidential treatment, so the flag rides on the quarter. + var point = Read(ThreeQuarterLine); + + Assert.AreEqual(new[] { false, true, false }, + point.Holdings.Select(holding => holding.ConfidentialOmitted).ToArray()); + } + + [Test] + public void ReleaseIsStampedAfterTheQuarterItReports() + { + // The lag from the reported quarter to publication is the product here: measured across + // 11,761 filings it runs from 0 to 6,596 days with a median of 42. Whatever its size, + // the release can never precede the quarter it reports. + var point = Read(FullLine); + + Assert.GreaterOrEqual(point.Time.Date, point.Holdings.Single().PeriodEnd); + } + + [Test] + public void AVeryLongFilingLagIsLegitimateData() + { + // Amendments for old quarters keep arriving; the measured worst case is 6,596 days. + // A row like this is late, not corrupt, and must parse. + var point = Read("20260122 03:00,1,20071231,3,1000,50000,,,,,,0"); + var quarter = point.Holdings.Single(); + + Assert.AreEqual(new DateTime(2026, 1, 22, 3, 0, 0), point.Time); + Assert.AreEqual(new DateTime(2007, 12, 31), quarter.PeriodEnd); + Assert.Greater((point.Time.Date - quarter.PeriodEnd).TotalDays, 6000); + } + + [Test] + public void AZeroDayFilingLagIsLegitimateData() + { + // The measured minimum is zero: filed on the reported quarter end itself. + var point = Read("20240630 03:00,1,20240630,3,1000,50000,,,,,,0"); + + Assert.AreEqual(point.Time.Date, point.Holdings.Single().PeriodEnd); + } + + [TestCase("20240331", 2024, 3, 31)] + [TestCase("20240630", 2024, 6, 30)] + [TestCase("20240930", 2024, 9, 30)] + [TestCase("20241231", 2024, 12, 31)] + public void PeriodEndComesFromTheFileNotFromTime(string periodEnd, int year, int month, int day) + { + // Deriving the quarter from the release date would be wrong: one publication day + // carries filings for several different reported quarters, so it has to be a column. + var point = Read($"20260515 03:00,1,{periodEnd},3,1000,50000,,,,,,0"); + + Assert.AreEqual(new DateTime(year, month, day), point.Holdings.Single().PeriodEnd); + Assert.AreEqual(new DateTime(2026, 5, 15, 3, 0, 0), point.Time); + } + + [TestCase("20200331", "2020Q1")] + [TestCase("20200630", "2020Q2")] + [TestCase("20200930", "2020Q3")] + [TestCase("20201231", "2020Q4")] + public void QuarterLabelsTheReportedPeriod(string periodEnd, string expected) + { + // The label is derived from PeriodEnd and never stored, so the two cannot disagree. The + // year comes first so that sorting the label sorts the quarters. + Assert.AreEqual(expected, Read($"20260515 03:00,1,{periodEnd},3,1000,50000,,,,,,0").Latest.Quarter); + } + + [Test] + public void ATruncatedLineIsSkippedRatherThanThrown() + { + // An exception out of Reader ends the algorithm, so a short line has to come back as + // null. It is the whole point of the guard: one malformed row must not take a live + // strategy down. + Assert.IsNull(Read("20240214 03:00,1,20231231,4812")); + } + + [TestCase("20240214 03:00,3,20231231,4812,1,1,0,0,0,0,0,0", TestName = "three quarters promised, one written")] + [TestCase("20240214 03:00,0,20231231,4812,1,1,0,0,0,0,0,0", TestName = "no quarters at all")] + [TestCase("20240214 03:00,-2,20231231,4812,1,1,0,0,0,0,0,0", TestName = "a negative count")] + [TestCase("20240214 03:00,many,20231231,4812,1,1,0,0,0,0,0,0", TestName = "a count that is not a number")] + public void AQuarterCountThatDoesNotMatchTheLineIsSkipped(string line) + { + // The count is what the group width is derived from, so a count the columns cannot + // support would read a quarter out of the middle of another one. Same rule as a + // truncated line: null rather than an exception, and the file carries on. + Assert.IsNull(Read(line)); + } + + [Test] + public void AnExtraColumnStillParses() + { + // The guard is "fewer than", not "not equal to". A column appended in a later revision + // of the file must leave every existing one readable rather than mute the dataset. + var point = Read(FullLine + ",99"); + + Assert.IsNotNull(point); + Assert.AreEqual(4812m, point.Holdings.Single().Holders); + } + + [Test] + public void AnExtraColumnPerQuarterStillParses() + { + // The group width is derived from the line and its count rather than assumed, so a + // measure appended to every group leaves each existing quarter where the reader expects + // it instead of shifting the ones behind it. + var point = Read( + "20240515 03:00,2," + + "20230930,4800,48000,480000,0,0,0,0,0,0,99," + + "20231231,12,120,1200,0,0,0,0,0,1,98"); + + Assert.IsNotNull(point); + Assert.AreEqual(2, point.Holdings.Count); + Assert.AreEqual(new DateTime(2023, 9, 30), point.Holdings[0].PeriodEnd); + Assert.AreEqual(4800m, point.Holdings[0].Holders); + Assert.AreEqual(new DateTime(2023, 12, 31), point.Holdings[1].PeriodEnd); + Assert.AreEqual(12m, point.Holdings[1].Holders); + Assert.IsTrue(point.Holdings[1].ConfidentialOmitted, "the flag is still the last column of its own group"); + } + + [Test] + public void NonNumericMeasureThrows() + { + // Column 0 is a well formed release stamp, so the line reaches the decimal parse rather + // than failing on the timestamp before it. + Assert.Throws( + () => Read("20240214 03:00,1,20231231,4812,not-a-number,3210456789012,,,,,,0")); + } + + [Test] + public void BadTimeThrows() + { + Assert.Throws( + () => Read("2023-10-01,1,20231231,4812,10589324611,3210456789012,,,,,,0")); + } + + [Test] + public void GetSourceIsTheLocalPerSecurityFile() + { + var source = new SEC13F().GetSource(Config(), new DateTime(2024, 2, 14), false); + + Assert.AreEqual(SubscriptionTransportMedium.LocalFile, source.TransportMedium); + Assert.AreEqual(FileFormat.Csv, source.Format); + Assert.IsTrue( + Normalize(source.Source).EndsWith("alternative/sec/13f/aapl.csv", StringComparison.Ordinal), + $"Unexpected source '{source.Source}'"); + } + + [Test] + public void ClassificationMatchesTheSpec() + { + var instance = new SEC13F(); + + Assert.IsTrue(instance.IsSparseData()); + Assert.IsTrue(instance.RequiresMapping()); + Assert.AreEqual(Resolution.Daily, instance.DefaultResolution()); + Assert.AreEqual(1, instance.SupportedResolutions().Count); + Assert.AreEqual(Resolution.Daily, instance.SupportedResolutions()[0]); + Assert.AreEqual(TimeZones.NewYork, instance.DataTimeZone()); + } + + [Test] + public void ToStringNamesEveryQuarterItCarries() + { + var text = Read(ThreeQuarterLine).ToString(); + + Assert.IsFalse(string.IsNullOrWhiteSpace(text)); + foreach (var quarter in new[] { "2023Q2", "2023Q3", "2023Q4" }) + { + Assert.IsTrue(text.Contains(quarter, StringComparison.Ordinal), $"'{text}' does not name {quarter}"); + } + } + + [Test] + public void AProtobufRoundTripKeepsEveryQuarter() + { + // How the live data distributor moves a point: it registers the custom type as a + // BaseData sub type, in ProtobufSubTypes here, and serializes through the BaseData + // branch. A quarter lost on the way would reach a live algorithm as an absent reading + // while the same algorithm saw it in a backtest, the worst way to lose data. + var original = Read(ThreeQuarterLine); + var bytes = ((BaseData)original).ProtobufSerialize(new Guid()); + + using var stream = new MemoryStream(bytes); + var deserialized = Serializer + .DeserializeItems(stream, PrefixStyle.Base128, 1) + .Single() as SEC13F; + + Assert.IsNotNull(deserialized, "the point did not come back as a SEC13F"); + Assert.AreEqual(original.Time, deserialized.Time); + Assert.AreEqual(original.Holdings.Count, deserialized.Holdings.Count); + + for (var i = 0; i < original.Holdings.Count; i++) + { + var expected = original.Holdings[i]; + var actual = deserialized.Holdings[i]; + + Assert.AreEqual(expected.PeriodEnd, actual.PeriodEnd, $"quarter {i}: PeriodEnd"); + Assert.AreEqual(expected.Quarter, actual.Quarter, $"quarter {i}: Quarter"); + Assert.AreEqual(expected.Holders, actual.Holders, $"quarter {i}: Holders"); + Assert.AreEqual(expected.Shares, actual.Shares, $"quarter {i}: Shares"); + Assert.AreEqual(expected.HoldingValue, actual.HoldingValue, $"quarter {i}: HoldingValue"); + Assert.AreEqual(expected.CallShares, actual.CallShares, $"quarter {i}: CallShares"); + Assert.AreEqual(expected.PutShares, actual.PutShares, $"quarter {i}: PutShares"); + Assert.AreEqual(expected.PrincipalValue, actual.PrincipalValue, $"quarter {i}: PrincipalValue"); + Assert.AreEqual(expected.VotingSole, actual.VotingSole, $"quarter {i}: VotingSole"); + Assert.AreEqual(expected.VotingShared, actual.VotingShared, $"quarter {i}: VotingShared"); + Assert.AreEqual(expected.ConfidentialOmitted, actual.ConfidentialOmitted, $"quarter {i}: ConfidentialOmitted"); + } + + // The readings derived from the quarters have to come back with them. + Assert.AreEqual(original.MostReported.PeriodEnd, deserialized.MostReported.PeriodEnd); + Assert.AreEqual(original.Latest.PeriodEnd, deserialized.Latest.PeriodEnd); + } + + [Test] + public void EveryProcessedRowParses() + { + var directory = SEC13FTestPaths.HoldingsDirectory; + if (directory == null || !Directory.Exists(directory)) + { + Assert.Ignore($"No processed output at {SEC13FTestPaths.Describe(directory)}"); + } + + var files = Directory.GetFiles(directory, "*.csv", SearchOption.TopDirectoryOnly); + if (files.Length == 0) + { + Assert.Ignore($"No per-security files in {directory}"); + } + + // The quarter count is the second column of a per-security line. + SEC13FTestPaths.RequirePackedLayout(files, 1); + + var instance = new SEC13F(); + var points = 0; + var quarters = 0; + + foreach (var file in files) + { + var ticker = Path.GetFileNameWithoutExtension(file).ToUpperInvariant(); + var config = Config(ticker); + var previous = DateTime.MinValue; + + foreach (var line in File.ReadLines(file)) + { + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var point = instance.Reader(config, line, DateTime.UtcNow, false) as SEC13F; + + Assert.IsNotNull(point, $"{ticker}: '{line}' did not parse"); + Assert.AreEqual(point.Time, point.EndTime, $"{ticker}: EndTime is not Time on {point.Time:yyyy-MM-dd}"); + Assert.IsNotEmpty(point.Holdings, $"{ticker}: a point carrying no quarter on {point.Time:yyyy-MM-dd}"); + Assert.AreEqual(point.MostReported.HoldingValue ?? 0m, point.Value, + $"{ticker}: Value does not mirror the most reported quarter"); + + var period = DateTime.MinValue; + foreach (var holding in point.Holdings) + { + Assert.AreEqual(point.Symbol, holding.Symbol, $"{ticker}: a quarter names another security"); + Assert.GreaterOrEqual(point.Time.Date, holding.PeriodEnd, + $"{ticker}: released before the quarter it reports"); + + // Written oldest first and never twice, which is what makes Latest the + // newest quarter rather than whichever one the file happened to end on. + Assert.Greater(holding.PeriodEnd, period, + $"{ticker}: quarters out of order on {point.Time:yyyy-MM-dd}"); + period = holding.PeriodEnd; + quarters++; + } + + // LEAN's SubscriptionDataReader silently drops a point whose timestamp moves + // backwards, so a file out of chronological order loses rows with nothing in + // the log to show for it. Accumulating groups by quarter, which is NOT the + // order the file has to ship in, so this is worth asserting on real output. + // One line per release, so a repeated timestamp is a dropped point too. + Assert.Greater(point.Time, previous, + $"{ticker}: {point.Time:yyyy-MM-dd HH:mm} follows {previous:yyyy-MM-dd HH:mm}"); + previous = point.Time; + points++; + } + } + + TestContext.Progress.WriteLine( + $"13f: {files.Length} per-security files, {points} points, {quarters} quarters"); + Assert.Greater(points, 0); + } + + [Test] + public void ThePointsProtobufContractIsCompleteAndUnique() + { + // DeclaredOnly: BaseData numbers its own members from 1, and those are LEAN's to own. + var members = ProtoMembers(typeof(SEC13F)); + + Assert.IsNotNull(typeof(SEC13F).GetCustomAttribute(), "ProtoContract is missing"); + Assert.Greater(members.Count, 0, "no ProtoMember attributes found"); + Assert.AreEqual(members.Count, members.Distinct().Count(), "duplicate ProtoMember numbers"); + Assert.GreaterOrEqual(members.Min(), 10, "member numbers start at 10, BaseData reserves the low ones"); + } + + [Test] + public void TheQuartersProtobufContractIsCompleteAndUnique() + { + // A quarter is deliberately not a BaseData, so that it can ride inside the point. It + // owns its whole number space and starts at one. + var members = ProtoMembers(typeof(SEC13FHolding)); + + Assert.IsNotNull(typeof(SEC13FHolding).GetCustomAttribute(), "ProtoContract is missing"); + Assert.Greater(members.Count, 0, "no ProtoMember attributes found"); + Assert.AreEqual(members.Count, members.Distinct().Count(), "duplicate ProtoMember numbers"); + } + + [TestCase(typeof(SEC13F))] + [TestCase(typeof(SEC13FHolding))] + public void EveryStoredPropertyCarriesAProtoMember(Type type) + { + // A property added later without a member number would serialize as nothing, which is + // indistinguishable from a real absent reading. Symbol is the one exception, the same + // one BaseData itself makes: the distributor sends the symbol alongside the point + // rather than inside it. + var unnumbered = type + .GetProperties(BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly) + .Where(property => property.CanRead && property.CanWrite && property.GetIndexParameters().Length == 0) + .Where(property => property.Name != nameof(SEC13FHolding.Symbol)) + .Where(property => property.GetCustomAttribute() == null) + .Select(property => property.Name) + .ToList(); + + Assert.IsEmpty(unnumbered, $"{type.Name}: stored properties with no ProtoMember: {string.Join(", ", unnumbered)}"); + } + + private static List ProtoMembers(Type type) + { + return type.GetProperties(BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly) + .Select(property => property.GetCustomAttribute()) + .Where(attribute => attribute != null) + .Select(attribute => attribute.Tag) + .ToList(); + } + + /// + /// Compares two instances member by member, so that a property added to a class and + /// forgotten in its Clone() is caught without a hand-written list to keep up to date. + /// + internal static void AssertPropertiesAndFieldsAreEqual(object expected, object actual) + { + foreach (var property in expected.GetType().GetProperties()) + { + AssertValueWasCopied(property.GetValue(expected), property.GetValue(actual), $"Property '{property.Name}'"); + } + + foreach (var field in expected.GetType().GetFields()) + { + AssertValueWasCopied(field.GetValue(expected), field.GetValue(actual), $"Field '{field.Name}'"); + } + } + + /// + /// Compares one member. A point and a quarter are compared member by member rather than by + /// reference: a clone that shared them would let the next release mutate a point the + /// algorithm has already read, so equality here can never mean the same instance. + /// + private static void AssertValueWasCopied(object expected, object actual, string member) + { + switch (expected) + { + case SEC13F: + case SEC13FHolding: + Assert.IsNotNull(actual, $"{member} was not copied"); + Assert.AreEqual(expected.GetType(), actual.GetType(), $"{member} changed type"); + AssertPropertiesAndFieldsAreEqual(expected, actual); + break; + + case IEnumerable items: + var copies = (actual as IEnumerable)?.ToList(); + Assert.IsNotNull(copies, $"{member} was not copied"); + + var originals = items.ToList(); + Assert.AreEqual(originals.Count, copies.Count, $"{member} changed length"); + for (var i = 0; i < originals.Count; i++) + { + AssertValueWasCopied(originals[i], copies[i], $"{member}[{i}]"); + } + + break; + + default: + Assert.AreEqual(expected, actual, $"{member} was not copied"); + break; + } + } + + internal static string Normalize(string path) => path.Replace('\\', '/'); + } + + /// + /// Locates the processed 13F output produced by Phase 3. The tests that use it skip cleanly + /// while the folder does not exist yet. + /// + internal static class SEC13FTestPaths + { + /// Root of the processed output, or null when the repository root cannot be found. + public static string HoldingsDirectory + { + get + { + var root = Environment.GetEnvironmentVariable("SEC_13F_OUTPUT"); + if (!string.IsNullOrWhiteSpace(root)) + { + return root; + } + + var repository = RepositoryRoot(); + return repository == null + ? null + : Path.Combine(repository, "output", "alternative", "sec", "13f"); + } + } + + public static string UniverseDirectory + { + get + { + var holdings = HoldingsDirectory; + return holdings == null ? null : Path.Combine(holdings, "universe"); + } + } + + public static string Describe(string directory) => directory ?? ""; + + /// + /// Skips the caller when the sample output on disk still carries one line per reported + /// quarter, the layout that shipped before a release became a single point. Such a sample + /// is an artefact of an older processor run rather than a parsing failure, and the only fix + /// is to run the processor again. Everything else is left to fail in the caller. + /// + /// The sample files about to be read. + /// Index of the quarter count, which the old layout does not have. + public static void RequirePackedLayout(IEnumerable files, int countColumn) + { + var stale = files.Where(file => IsPrePackingLayout(FirstLine(file), countColumn)).ToList(); + if (stale.Count == 0) + { + return; + } + + Assert.Ignore( + $"{stale.Count} sample file(s) under output/ are still in the pre-packing layout, one line per " + + $"reported quarter rather than one per release. Run the processor again to refresh them. " + + $"First: {stale[0]}"); + } + + private static string FirstLine(string file) + { + return File.ReadLines(file).FirstOrDefault(line => !string.IsNullOrWhiteSpace(line)); + } + + // The column that now holds the quarter count used to hold the reported quarter itself, and + // a count never looks like a date, so the two layouts cannot be confused for each other. + private static bool IsPrePackingLayout(string line, int countColumn) + { + var csv = line?.Split(','); + return csv != null + && csv.Length > countColumn + && DateTime.TryParseExact(csv[countColumn], "yyyyMMdd", CultureInfo.InvariantCulture, + DateTimeStyles.None, out _); + } + + // The test binaries live in tests/bin//, so walk up to the folder that + // holds the data source project file. + private static string RepositoryRoot() + { + var directory = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + + while (directory != null) + { + if (File.Exists(Path.Combine(directory.FullName, "QuantConnect.DataSource.csproj"))) + { + return directory.FullName; + } + + directory = directory.Parent; + } + + return null; + } + } +} diff --git a/tests/SEC13FUniverseTests.cs b/tests/SEC13FUniverseTests.cs new file mode 100644 index 0000000..92ad6b4 --- /dev/null +++ b/tests/SEC13FUniverseTests.cs @@ -0,0 +1,405 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using NUnit.Framework; +using QuantConnect.Data; +using QuantConnect.DataSource; + +namespace QuantConnect.DataLibrary.Tests +{ + /// + /// Unit tests for , the release-date view of the dataset. Every case + /// parses an in-line sample line, so the fixture needs no data on disk and is safe to run in CI. + /// The one test that touches the processed output is skipped when that folder is not there yet. + /// + [TestFixture] + public class SEC13FUniverseTests + { + private const string AppleSid = "AAPL R735QTJ8XC9X"; + + // Frozen line layout: the identifier, the ticker, how many quarters the security carries, + // and then that many groups of ten columns, oldest quarter first: + // PeriodEnd,Holders,Shares,HoldingValue,CallShares,PutShares,PrincipalValue,VotingSole,VotingShared,ConfidentialOmitted + private const string FullLine = + AppleSid + ",AAPL,1,20231231,4812,10589324611,3210456789012,152340000,98760000,4500000,9012345678,1234567890,1"; + + private const string EmptyValuesLine = AppleSid + ",AAPL,1,20231231,,,,,,,,,0"; + + // The usual shape of a universe line during a handover: the finished quarter, which holds + // the managers, and the one filling in behind it. A security never carries more than two. + private const string TwoQuarterLine = + AppleSid + ",AAPL,2," + + "20230930,4800,48000,480000,0,0,0,0,0,1," + + "20231231,12,120,1200,0,0,0,0,0,0"; + + // Same release date, an amendment for a much older quarter. PeriodEnd cannot be derived. + private const string LateAmendmentLine = AppleSid + ",AAPL,1,20180630,7,5000,120000,,,,,,0"; + + private static readonly DateTime ReleaseDate = new DateTime(2024, 2, 14); + + private static SubscriptionDataConfig Config() + { + return new SubscriptionDataConfig( + typeof(SEC13FUniverse), + Symbol.Create("AAPL", SecurityType.Base, Market.USA), + Resolution.Daily, + TimeZones.NewYork, + TimeZones.NewYork, + false, false, false); + } + + /// + /// One universe line, read the way the folding collection reads it: the line becomes the + /// security's own data point rather than a record of its own. + /// + private static SEC13F Read(string line, DateTime? date = null) + { + return new SEC13FUniverse().Reader(Config(), line, date ?? ReleaseDate, false) as SEC13F; + } + + [Test] + public void ReaderParsesEveryColumn() + { + var point = Read(FullLine); + + Assert.IsNotNull(point); + Assert.AreEqual(SecurityIdentifier.Parse(AppleSid), point.Symbol.ID); + Assert.AreEqual("AAPL", point.Symbol.Value); + Assert.AreEqual(1, point.Holdings.Count); + + var quarter = point.Holdings[0]; + Assert.AreEqual(new DateTime(2023, 12, 31), quarter.PeriodEnd); + Assert.AreEqual("2023Q4", quarter.Quarter); + Assert.AreEqual(4812m, quarter.Holders); + Assert.AreEqual(10589324611m, quarter.Shares); + Assert.AreEqual(3210456789012m, quarter.HoldingValue); + Assert.AreEqual(152340000m, quarter.CallShares); + Assert.AreEqual(98760000m, quarter.PutShares); + Assert.AreEqual(4500000m, quarter.PrincipalValue); + Assert.AreEqual(9012345678m, quarter.VotingSole); + Assert.AreEqual(1234567890m, quarter.VotingShared); + Assert.IsTrue(quarter.ConfidentialOmitted); + Assert.AreEqual(3210456789012m, point.Value); + } + + [Test] + public void ALineIsOneRecordPerSecurityWithItsQuartersInside() + { + // The selection function is handed one record per security, not one per quarter: a + // security with two live quarters would otherwise appear twice in the same file and the + // second reading would silently win. + var point = Read(TwoQuarterLine); + + Assert.IsNotNull(point); + Assert.AreEqual(2, point.Holdings.Count); + Assert.AreEqual(new[] { "2023Q3", "2023Q4" }, point.Holdings.Select(holding => holding.Quarter).ToArray()); + Assert.AreEqual(new[] { 4800m, 12m }, point.Holdings.Select(holding => holding.Holders).ToArray()); + + // The finished quarter holds the managers while the new one fills in, so it is the one + // a cross-section is ranked on. The newest is still reachable through Latest. + Assert.AreEqual(new DateTime(2023, 9, 30), point.MostReported.PeriodEnd); + Assert.AreEqual(new DateTime(2023, 12, 31), point.Latest.PeriodEnd); + Assert.AreEqual(480000m, point.Value); + } + + [Test] + public void SymbolRoundTripsThroughSecurityIdentifier() + { + var point = Read(TwoQuarterLine); + + Assert.AreEqual(AppleSid, point.Symbol.ID.ToString()); + Assert.AreEqual(SecurityType.Equity, point.Symbol.SecurityType); + Assert.IsTrue(point.Holdings.All(holding => holding.Symbol == point.Symbol), + "every quarter names the security its line identified"); + } + + [Test] + public void ThePointSpansTheFileDateAndEndsAtTheReleaseTime() + { + // The file name is the release date, so the stamp comes from the requested date and + // never from a column: the point spans that day and becomes available at the release + // time, the same instant the per-security lines carry. Ending the point at the release + // rather than starting it there is what the engine expects of a universe file, and a + // point that started there would be walked a day at a time from a three hour offset. + var date = new DateTime(2021, 8, 16); + var point = Read(FullLine, date); + + Assert.AreEqual(date, point.Time); + Assert.AreEqual(date.Add(SEC13F.ReleaseTimeOfDay), point.EndTime); + } + + [Test] + public void TheCollectionIsAvailableAtTheReleaseTimeOfItsPoints() + { + // The engine builds the collection around the points it folded, taking their Time and + // EndTime, so the release time reaches the selection through them rather than through + // an override of its own. A clone has to carry it too, since the engine clones the + // collection on its way to the algorithm. + var point = Read(FullLine, ReleaseDate); + var collection = new SEC13FUniverse + { + Symbol = Symbol.Create("SEC13F-USA", SecurityType.Base, Market.USA), + Time = point.Time, + EndTime = point.EndTime, + Data = new List { point } + }; + + Assert.AreEqual(ReleaseDate.Add(SEC13F.ReleaseTimeOfDay), collection.EndTime); + Assert.Greater(collection.EndTime, collection.Time); + Assert.AreEqual(collection.EndTime, ((SEC13FUniverse)collection.Clone()).EndTime); + } + + [Test] + public void PeriodEndComesFromTheFileNotFromTime() + { + // One release date carries filings for several reported quarters, so PeriodEnd must be + // read, not derived. Deriving it from Time would have given 2024-03-31 here. + var current = Read(FullLine); + var amendment = Read(LateAmendmentLine); + + Assert.AreEqual(current.Time, amendment.Time); + Assert.AreEqual(new DateTime(2023, 12, 31), current.Latest.PeriodEnd); + Assert.AreEqual(new DateTime(2018, 6, 30), amendment.Latest.PeriodEnd); + Assert.AreNotEqual(current.Latest.PeriodEnd, amendment.Latest.PeriodEnd); + } + + [Test] + public void CloneCopiesEveryProperty() + { + var original = new SEC13FUniverse + { + Symbol = Symbol.Create("AAPL", SecurityType.Base, Market.USA), + Time = ReleaseDate, + Data = new List { Read(TwoQuarterLine), Read(LateAmendmentLine) } + }; + + var clone = original.Clone() as SEC13FUniverse; + + Assert.IsNotNull(clone); + // Reflection rather than a hand-written list: a property added to the class but forgotten + // in Clone() is exactly the bug this test has to catch, and a hand-written list cannot. + SEC13FTests.AssertPropertiesAndFieldsAreEqual(original, clone); + } + + [Test] + public void ValueMirrorsTheMostReportedHoldingValue() + { + Assert.AreEqual(3210456789012m, Read(FullLine).Value); + // A withheld HoldingValue leaves the point at zero rather than throwing or carrying null. + Assert.AreEqual(0m, Read(EmptyValuesLine).Value); + } + + [Test] + public void EmptyNumericColumnsBecomeNullNotZero() + { + var quarter = Read(EmptyValuesLine)?.Holdings.SingleOrDefault(); + + Assert.IsNotNull(quarter); + Assert.IsNull(quarter.Holders); + Assert.IsNull(quarter.Shares); + Assert.IsNull(quarter.HoldingValue); + Assert.IsNull(quarter.CallShares); + Assert.IsNull(quarter.PutShares); + Assert.IsNull(quarter.PrincipalValue); + Assert.IsNull(quarter.VotingSole); + Assert.IsNull(quarter.VotingShared); + } + + [TestCase("1", true)] + [TestCase("0", false)] + [TestCase("", false)] + public void ConfidentialOmittedParsesFlag(string flag, bool expected) + { + var line = AppleSid + $",AAPL,1,20231231,4812,10589324611,3210456789012,,,,,,{flag}"; + + Assert.AreEqual(expected, Read(line).Holdings.Single().ConfidentialOmitted); + } + + [Test] + public void ATruncatedLineIsSkippedRatherThanThrown() + { + // An exception out of Reader ends the algorithm, so a short line has to come back as + // null. It is the whole point of the guard: one malformed row must not take a live + // strategy down. + Assert.IsNull(Read(AppleSid + ",AAPL,1,20231231")); + Assert.IsNull(Read(AppleSid), "a line with nothing but an identifier"); + } + + [Test] + public void AQuarterCountThatDoesNotMatchTheLineIsSkipped() + { + // The count is what the group width is derived from, so a count the columns cannot + // support would read a quarter out of the middle of another one. + Assert.IsNull(Read(AppleSid + ",AAPL,2,20231231,4812,10589324611,3210456789012,0,0,0,0,0,0")); + } + + [Test] + public void AnExtraColumnStillParses() + { + // The guard is "fewer than", not "not equal to". A column appended in a later revision + // of the file must leave every existing one readable rather than mute the dataset. + var point = Read(FullLine + ",99"); + + Assert.IsNotNull(point); + Assert.AreEqual(4812m, point.Holdings.Single().Holders); + } + + [Test] + public void AnExtraColumnPerQuarterStillParses() + { + // The group width is derived from the line and its count, so a measure appended to + // every group leaves each existing quarter where the reader expects it. + var point = Read( + AppleSid + ",AAPL,2," + + "20230930,4800,48000,480000,0,0,0,0,0,0,99," + + "20231231,12,120,1200,0,0,0,0,0,1,98"); + + Assert.AreEqual(2, point.Holdings.Count); + Assert.AreEqual(new DateTime(2023, 9, 30), point.Holdings[0].PeriodEnd); + Assert.AreEqual(4800m, point.Holdings[0].Holders); + Assert.AreEqual(new DateTime(2023, 12, 31), point.Holdings[1].PeriodEnd); + Assert.IsTrue(point.Holdings[1].ConfidentialOmitted, "the flag is still the last column of its own group"); + } + + [Test] + public void NonNumericMeasureThrows() + { + Assert.Throws( + () => Read(AppleSid + ",AAPL,1,20231231,4812,10589324611,not-a-number,,,,,,0")); + } + + [Test] + public void BadPeriodEndThrows() + { + Assert.Throws( + () => Read(AppleSid + ",AAPL,1,2023-12-31,4812,10589324611,3210456789012,,,,,,0")); + } + + [Test] + public void BadSecurityIdentifierThrows() + { + Assert.Catch( + () => Read("not-a-sid,AAPL,1,20231231,4812,10589324611,3210456789012,,,,,,0")); + } + + [Test] + public void GetSourceIsTheLocalReleaseDateFile() + { + var source = new SEC13FUniverse().GetSource(Config(), ReleaseDate, false); + + Assert.AreEqual(SubscriptionTransportMedium.LocalFile, source.TransportMedium); + Assert.AreEqual(FileFormat.FoldingCollection, source.Format); + Assert.IsTrue( + SEC13FTests.Normalize(source.Source) + .EndsWith("alternative/sec/13f/universe/20240214.csv", StringComparison.Ordinal), + $"Unexpected source '{source.Source}'"); + } + + [Test] + public void ClassificationMatchesTheSpec() + { + var instance = new SEC13FUniverse(); + + Assert.IsTrue(instance.IsSparseData()); + Assert.AreEqual(Resolution.Daily, instance.DefaultResolution()); + Assert.AreEqual(1, instance.SupportedResolutions().Count); + Assert.AreEqual(Resolution.Daily, instance.SupportedResolutions()[0]); + Assert.AreEqual(TimeZones.NewYork, instance.DataTimeZone()); + } + + [Test] + public void ToStringIsNotEmpty() + { + Assert.IsFalse(string.IsNullOrWhiteSpace(Read(TwoQuarterLine).ToString())); + } + + [Test] + public void EveryProcessedUniverseRowParses() + { + var directory = SEC13FTestPaths.UniverseDirectory; + if (directory == null || !Directory.Exists(directory)) + { + Assert.Ignore($"No processed universe output at {SEC13FTestPaths.Describe(directory)}"); + } + + var files = Directory.GetFiles(directory, "*.csv", SearchOption.TopDirectoryOnly); + if (files.Length == 0) + { + Assert.Ignore($"No universe files in {directory}"); + } + + // The quarter count is the third column of a universe line, after identifier and ticker. + SEC13FTestPaths.RequirePackedLayout(files, 2); + + var instance = new SEC13FUniverse(); + var config = Config(); + var rows = 0; + var quarters = 0; + + foreach (var file in files) + { + var name = Path.GetFileNameWithoutExtension(file); + Assert.IsTrue( + DateTime.TryParseExact(name, "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var date), + $"Universe file '{name}' is not named yyyyMMdd"); + + var securities = new HashSet(); + + foreach (var line in File.ReadLines(file)) + { + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var point = instance.Reader(config, line, date, false) as SEC13F; + + Assert.IsNotNull(point, $"{name}: '{line}' did not parse"); + Assert.AreEqual(date, point.Time, $"{name}: Time is not the file date"); + Assert.AreEqual(date.Add(SEC13F.ReleaseTimeOfDay), point.EndTime, + $"{name}: EndTime is not the release time"); + Assert.IsNotEmpty(point.Holdings, $"{name}: a security carrying no quarter"); + Assert.LessOrEqual(point.Holdings.Count, 2, + $"{name}: {point.Symbol.Value} carries more than the two live quarters"); + Assert.AreEqual(point.MostReported.HoldingValue ?? 0m, point.Value, + $"{name}: Value does not mirror the most reported quarter"); + + // One record per security is the whole point of packing the quarters together: + // a second line for the same name would quietly replace the first. + Assert.IsTrue(securities.Add(point.Symbol), $"{name}: {point.Symbol.Value} appears twice"); + + var period = DateTime.MinValue; + foreach (var holding in point.Holdings) + { + Assert.LessOrEqual(holding.PeriodEnd, date, $"{name}: a quarter ends after the release date"); + Assert.Greater(holding.PeriodEnd, period, $"{name}: {point.Symbol.Value} quarters out of order"); + period = holding.PeriodEnd; + quarters++; + } + + rows++; + } + } + + TestContext.Progress.WriteLine($"13f universe: {files.Length} files, {rows} securities, {quarters} quarters"); + Assert.Greater(rows, 0); + } + } +} diff --git a/tests/SECTests.cs b/tests/SECTests.cs index 6f93ce5..c647d6a 100644 --- a/tests/SECTests.cs +++ b/tests/SECTests.cs @@ -19,7 +19,6 @@ using System.IO; using System.Linq; using ProtoBuf; -using ProtoBuf.Meta; using Newtonsoft.Json; using NUnit.Framework; using QuantConnect.Data; @@ -47,7 +46,9 @@ public void ProtobufRoundTrip() var expected = CreateNewInstance(); var type = expected.GetType(); - RuntimeTypeModel.Default[typeof(BaseData)].AddSubType(2000, type); + // The sub type is registered once for the whole assembly, in ProtobufSubTypes: the + // model freezes as soon as anything is serialized, so a registration here would throw + // for whichever round trip test did not run first. using (var stream = new MemoryStream()) { diff --git a/tests/Tests.csproj b/tests/Tests.csproj index c3a31c6..6b8232f 100644 --- a/tests/Tests.csproj +++ b/tests/Tests.csproj @@ -11,6 +11,10 @@ + + + + PreserveNewest