Add the Form 13F institutional holdings dataset

.github/workflows/build.yml

@@ -37,7 +37,12 @@ jobs:
           path: Lean
 
       - name: Move Lean
-        run: mv Lean ../Lean
+        # The runners are self-hosted and ../Lean sits outside the checkout, so it survives
+        # between runs. Left in place, mv moves into it instead of renaming, and the step
+        # fails on whichever runner has already built this repo once.
+        run: |
+          rm -rf ../Lean
+          mv Lean ../Lean
 
       - name: Run build and tests
         run: |

DataProcessing/DataProcessing.csproj

@@ -16,4 +16,9 @@
       <ProjectReference Include="..\QuantConnect.DataSource.csproj" />
     </ItemGroup>
 
+    <!-- config.json is deliberately not copied to the output directory. The deployed job writes its
+         own alongside the binary and every key here is read through a Config.Get default, while a
+         copy in the output flows down the ProjectReference chain into the test assembly, where it
+         would select the 13f dataset and repoint the process-wide data folder for the whole run. -->
+
 </Project>

DataProcessing/Program.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -14,7 +14,9 @@
 */
 
 using QuantConnect.Configuration;
+using QuantConnect.DataSource;
 using QuantConnect.Logging;
+using QuantConnect.Util;
 using System;
 using System.Diagnostics;
 using System.Globalization;
@@ -23,11 +25,60 @@
 namespace QuantConnect.DataProcessing
 {
     /// <summary>
-    /// Console program to convert from raw SEC data to a formatted form usable by LEAN
+    /// Console program to convert from raw SEC data to a formatted form usable by LEAN.
+    ///
+    /// The repository ships two unrelated SEC datasets and the "dataset-name" config key selects
+    /// which one a run processes:
+    ///
+    ///  - "reports" (the default): the 10-K, 10-Q and 8-K filings, downloaded with
+    ///    <see cref="SECDataDownloader"/> and converted with <see cref="SECDataConverter"/>.
+    ///  - "13f" (<see cref="SEC13FDownloader"/>): Form 13F institutional holdings, aggregated per
+    ///    security from the quarterly structured data sets.
+    ///
+    /// The default keeps a job that sets no dataset-name doing exactly what it did before the key
+    /// existed.
     /// </summary>
     public class Program
     {
-        public static void Main()
+        /// <summary>
+        /// The "dataset-name" config value that selects the shipped SEC reports dataset, and the
+        /// value assumed when the key is not set.
+        /// </summary>
+        private const string ReportsDatasetName = "reports";
+
+        /// <summary>Config key that asks the 13F run to rebuild the whole history instead of one date.</summary>
+        internal const string RebuildHistoryKey = "sec-13f-rebuild-history";
+
+        /// <summary>
+        /// Entrypoint of the program. The exit code is returned rather than handed to
+        /// <see cref="Environment.Exit"/> from inside the work: that call does not unwind the stack,
+        /// so every finally block written for the failure paths would be skipped on all of them.
+        /// </summary>
+        /// <returns>Zero on success, one on any failure</returns>
+        public static int Main()
+        {
+            var dataset = Config.Get("dataset-name", ReportsDatasetName).Trim().ToLowerInvariant();
+
+            switch (dataset)
+            {
+                case ReportsDatasetName:
+                    return ProcessReports();
+
+                case SEC13FDownloader.DatasetName:
+                    return Process13F();
+
+                default:
+                    Log.Error($"DataProcessing.Main(): Unknown dataset-name '{dataset}'. Valid options: " +
+                              $"{ReportsDatasetName}, {SEC13FDownloader.DatasetName}");
+                    return 1;
+            }
+        }
+
+        /// <summary>
+        /// Downloads and converts the SEC reports dataset for the deployment date.
+        /// </summary>
+        /// <returns>Zero on success, one on any failure</returns>
+        private static int ProcessReports()
         {
             var processingDateValue = Environment.GetEnvironmentVariable("QC_DATAFLEET_DEPLOYMENT_DATE");
             var processingDate = DateTime.ParseExact(processingDateValue, "yyyyMMdd", CultureInfo.InvariantCulture);
@@ -62,10 +113,115 @@ public static void Main()
             catch (Exception e)
             {
                 Log.Error(e, $"DataProcessing.Main(): {processingDate} Exception while processing SEC data");
-                Environment.Exit(1);
+                return 1;
+            }
+
+            return 0;
+        }
+
+        /// <summary>
+        /// Downloads and converts the Form 13F institutional holdings dataset for the deployment date,
+        /// folding it into the published history, or rebuilds the whole history when asked to.
+        /// </summary>
+        /// <returns>Zero on success, one on any failure</returns>
+        private static int Process13F()
+        {
+            // Output root: {temp-output-directory}/alternative/sec. The downloader writes its report
+            // under the folder its data class names.
+            var destinationDirectory = Path.Combine(
+                Config.Get("temp-output-directory", "/temp-output-directory"),
+                "alternative",
+                SEC13FDownloader.VendorName);
+
+            // The published history an incremental run merges its rows into. The destination is
+            // handed to the job empty, so reading history back from there would find nothing.
+            var processedDataDirectory = Path.Combine(
+                Config.Get("processed-data-directory", Globals.DataFolder),
+                "alternative",
+                SEC13FDownloader.VendorName);
+
+            // Where downloads land. The job archives this folder after every run, as it does the
+            // reports dataset's feed archives, but does not restore it before the next.
+            var rawDataDirectory = Path.Combine(
+                Config.Get("raw-data-folder", "/raw"),
+                "alternative",
+                SEC13FDownloader.VendorName);
+
+            if (!TryParseDeploymentDate(out var deploymentDate))
+            {
+                return 1;
+            }
+
+            Log.Trace($"DataProcessing.Process13F(): writing {SEC13FDownloader.DatasetName} to {destinationDirectory}"
+                      + (deploymentDate == null ? " for the full history" : $" for {deploymentDate:yyyy-MM-dd}"));
+
+            var timer = Stopwatch.StartNew();
+            SEC13FDownloader downloader;
+            try
+            {
+                downloader = new SEC13FDownloader(destinationDirectory, processedDataDirectory, deploymentDate,
+                    rawDataDirectory);
+            }
+            catch (Exception err)
+            {
+                Log.Error(err, $"DataProcessing.Process13F(): The {SEC13FDownloader.DatasetName} downloader failed to be constructed");
+                return 1;
+            }
+
+            try
+            {
+                downloader.Run();
+
+                timer.Stop();
+                Log.Trace($"DataProcessing.Process13F(): Conversion finished in time {timer.Elapsed}");
+                return 0;
+            }
+            catch (Exception err)
+            {
+                Log.Error(err, $"DataProcessing.Process13F(): The {SEC13FDownloader.DatasetName} downloader exited unexpectedly");
+                return 1;
+            }
+            finally
+            {
+                downloader.DisposeSafely();
+            }
+        }
+
+        /// <summary>
+        /// Reads the deployment date the job runs for. A missing date is a misconfigured job rather
+        /// than a request for the whole history, which is asked for by name with
+        /// <see cref="RebuildHistoryKey"/>.
+        /// </summary>
+        /// <param name="deploymentDate">The date, or null when the run rebuilds the whole history</param>
+        /// <returns>True when the date is well formed, or absent with the rebuild asked for</returns>
+        internal static bool TryParseDeploymentDate(out DateTime? deploymentDate)
+        {
+            deploymentDate = null;
+
+            var raw = Environment.GetEnvironmentVariable("QC_DATAFLEET_DEPLOYMENT_DATE");
+            if (string.IsNullOrWhiteSpace(raw))
+            {
+                if (Config.GetBool(RebuildHistoryKey))
+                {
+                    return true;
+                }
+
+                Log.Error("DataProcessing.TryParseDeploymentDate(): QC_DATAFLEET_DEPLOYMENT_DATE is not set. Set it, or " +
+                          $"set \"{RebuildHistoryKey}\": true to rebuild the whole history");
+                return false;
+            }
+
+            // A malformed date must not quietly become a full history run: that would turn a daily
+            // job into a complete refetch of five gigabytes without anyone noticing.
+            if (!DateTime.TryParseExact(raw.Trim(), "yyyyMMdd", CultureInfo.InvariantCulture,
+                    DateTimeStyles.None, out var parsed))
+            {
+                Log.Error($"DataProcessing.TryParseDeploymentDate(): QC_DATAFLEET_DEPLOYMENT_DATE '{raw}' is not yyyyMMdd");
+                return false;
             }
 
-            Environment.Exit(0);
+            deploymentDate = parsed;
+            return true;
         }
     }
 }

DataProcessing/SEC13FClosePrices.cs

@@ -0,0 +1,94 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+
+namespace QuantConnect.DataProcessing
+{
+    /// <summary>
+    /// Raw closing prices from LEAN's coarse universe files, one per trading day, which is what a
+    /// reported VALUE over SSHPRNAMT is checked against. The close of the quarter's last trading day
+    /// is known before any filing for that quarter can be made, so the check needs no other filing
+    /// and gives the same answer on the day a filing is read as in a rebuild years later.
+    /// </summary>
+    internal sealed class SEC13FClosePrices
+    {
+        /// <summary>Days walked back from a quarter end for its last trading day: a long weekend and a holiday.</summary>
+        private const int DaysToLookBack = 7;
+
+        private readonly string _coarseDirectory;
+        private readonly Dictionary<DateTime, Dictionary<string, decimal>> _closesByDay = new();
+
+        public SEC13FClosePrices(string coarseDirectory)
+        {
+            _coarseDirectory = coarseDirectory;
+        }
+
+        /// <summary>Whether the coarse files are there at all, so the caller can say so once.</summary>
+        public bool Available => Directory.Exists(_coarseDirectory);
+
+        /// <summary>
+        /// The raw close of a security on the last trading day on or before the quarter end, or
+        /// null. Never a day after the filing date: a period typed years ahead would otherwise read
+        /// a price nobody had when the filing was made.
+        /// </summary>
+        public decimal? Close(SecurityIdentifier security, DateTime periodEnd, DateTime filingDate)
+        {
+            var last = periodEnd.Date < filingDate.Date ? periodEnd.Date : filingDate.Date;
+            for (var back = 0; back < DaysToLookBack; back++)
+            {
+                var closes = ClosesOn(last.AddDays(-back));
+                if (closes != null)
+                {
+                    // The last trading day decides: a security missing from it did not trade then.
+                    return closes.TryGetValue(security.ToString(), out var close) && close > 0m ? close : null;
+                }
+            }
+
+            return null;
+        }
+
+        /// <summary>The closes of one trading day, keyed by security identifier, or null when the day has no file.</summary>
+        private Dictionary<string, decimal> ClosesOn(DateTime day)
+        {
+            if (_closesByDay.TryGetValue(day, out var closes))
+            {
+                return closes;
+            }
+
+            var path = Path.Combine(_coarseDirectory, $"{day.ToString(DateFormat.EightCharacter, CultureInfo.InvariantCulture)}.csv");
+            if (File.Exists(path))
+            {
+                closes = new Dictionary<string, decimal>(StringComparer.Ordinal);
+
+                // sid, ticker, close, volume, dollar volume, has fundamentals, price factor, split factor
+                foreach (var line in File.ReadLines(path))
+                {
+                    var fields = line.Split(',');
+                    if (fields.Length > 2 && decimal.TryParse(fields[2], NumberStyles.Any, CultureInfo.InvariantCulture, out var close))
+                    {
+                        closes[fields[0]] = close;
+                    }
+                }
+            }
+
+            _closesByDay[day] = closes;
+            return closes;
+        }
+    }
+}