Skip to content

Commit

Permalink
Use Steam manifest format
Browse files Browse the repository at this point in the history
  • Loading branch information
@NicknineTheEagle @azuisleet
NicknineTheEagle authored and azuisleet committed Nov 15, 2024
1 parent 0fb803b commit 1c97b42
Show file tree
Hide file tree
Showing 3 changed files with 199 additions and 117 deletions.
180 changes: 66 additions & 114 deletions DepotDownloader/ContentDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -598,20 +598,20 @@ static async Task<DepotDownloadInfo> GetDepotInfo(uint depotId, uint appId, ulon
return new DepotDownloadInfo(depotId, appId, manifestId, branch, installDir, depotKey);
}

private class ChunkMatch(ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk)
private class ChunkMatch(DepotManifest.ChunkData oldChunk, DepotManifest.ChunkData newChunk)
{
public ProtoManifest.ChunkData OldChunk { get; } = oldChunk;
public ProtoManifest.ChunkData NewChunk { get; } = newChunk;
public DepotManifest.ChunkData OldChunk { get; } = oldChunk;
public DepotManifest.ChunkData NewChunk { get; } = newChunk;
}

private class DepotFilesData
{
public DepotDownloadInfo depotDownloadInfo;
public DepotDownloadCounter depotCounter;
public string stagingDir;
public ProtoManifest manifest;
public ProtoManifest previousManifest;
public List<ProtoManifest.FileData> filteredFiles;
public DepotManifest manifest;
public DepotManifest previousManifest;
public List<DepotManifest.FileData> filteredFiles;
public HashSet<string> allFileNames;
}

Expand Down Expand Up @@ -694,8 +694,8 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat

Console.WriteLine("Processing depot {0}", depot.DepotId);

ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;
DepotManifest oldManifest = null;
DepotManifest newManifest = null;
var configDir = Path.Combine(depot.InstallDir, CONFIG_DIR);

var lastManifestId = INVALID_MANIFEST_ID;
Expand All @@ -707,72 +707,28 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat

if (lastManifestId != INVALID_MANIFEST_ID)
{
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.DepotId, lastManifestId));

if (File.Exists(oldManifestFileName))
{
byte[] expectedChecksum;

try
{
expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}

oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out var currentChecksum);

if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))
{
// We only have to show this warning if the old manifest ID was different
if (lastManifestId != depot.ManifestId)
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId);
oldProtoManifest = null;
}
}
// We only have to show this warning if the old manifest ID was different
var badHashWarning = (lastManifestId != depot.ManifestId);
oldManifest = Util.LoadManifestFromFile(configDir, depot.DepotId, lastManifestId, badHashWarning);
}

if (lastManifestId == depot.ManifestId && oldProtoManifest != null)
if (lastManifestId == depot.ManifestId && oldManifest != null)
{
newProtoManifest = oldProtoManifest;
newManifest = oldManifest;
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.DepotId);
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.DepotId, depot.ManifestId));
if (newManifestFileName != null)
{
byte[] expectedChecksum;

try
{
expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha");
}
catch (IOException)
{
expectedChecksum = null;
}

newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out var currentChecksum);

if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
{
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.ManifestId);
newProtoManifest = null;
}
}
newManifest = Util.LoadManifestFromFile(configDir, depot.DepotId, depot.ManifestId, true);

if (newProtoManifest != null)
if (newManifest != null)
{
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.DepotId);
}
else
{
Console.Write("Downloading depot manifest... ");

DepotManifest depotManifest = null;
ulong manifestRequestCode = 0;
var manifestRequestCodeExpiration = DateTime.MinValue;

Expand Down Expand Up @@ -820,7 +776,7 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat
depot.ManifestId,
connection,
cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy");
depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(
newManifest = await cdnPool.CDNClient.DownloadManifestAsync(
depot.DepotId,
depot.ManifestId,
manifestRequestCode,
Expand Down Expand Up @@ -872,9 +828,9 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.DepotId, depot.ManifestId, e.Message);
}
} while (depotManifest == null);
} while (newManifest == null);

if (depotManifest == null)
if (newManifest == null)
{
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.ManifestId, depot.DepotId);
cts.Cancel();
Expand All @@ -883,28 +839,22 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat
// Throw the cancellation exception if requested so that this task is marked failed
cts.Token.ThrowIfCancellationRequested();


newProtoManifest = new ProtoManifest(depotManifest, depot.ManifestId);
newProtoManifest.SaveToFile(newManifestFileName, out var checksum);
File.WriteAllBytes(newManifestFileName + ".sha", checksum);

Util.SaveManifestToFile(configDir, newManifest);
Console.WriteLine(" Done!");
}
}

newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal));

Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newProtoManifest.CreationTime);
Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newManifest.CreationTime);

if (Config.DownloadManifestOnly)
{
DumpManifestToTextFile(depot, newProtoManifest);
DumpManifestToTextFile(depot, newManifest);
return null;
}

var stagingDir = Path.Combine(depot.InstallDir, STAGING_DIR);

var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
var filesAfterExclusions = newManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
var allFileNames = new HashSet<string>(filesAfterExclusions.Count);

// Pre-process
Expand Down Expand Up @@ -936,8 +886,8 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat
depotDownloadInfo = depot,
depotCounter = depotCounter,
stagingDir = stagingDir,
manifest = newProtoManifest,
previousManifest = oldProtoManifest,
manifest = newManifest,
previousManifest = oldManifest,
filteredFiles = filesAfterExclusions,
allFileNames = allFileNames
};
Expand All @@ -952,7 +902,7 @@ private static async Task DownloadSteam3AsyncDepotFiles(CancellationTokenSource
Console.WriteLine("Downloading depot {0}", depot.DepotId);

var files = depotFilesData.filteredFiles.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
var networkChunkQueue = new ConcurrentQueue<(FileStreamData fileStreamData, ProtoManifest.FileData fileData, ProtoManifest.ChunkData chunk)>();
var networkChunkQueue = new ConcurrentQueue<(FileStreamData fileStreamData, DepotManifest.FileData fileData, DepotManifest.ChunkData chunk)>();

await Util.InvokeAsync(
files.Select(file => new Func<Task>(async () =>
Expand Down Expand Up @@ -1006,16 +956,16 @@ private static void DownloadSteam3AsyncDepotFile(
CancellationTokenSource cts,
GlobalDownloadCounter downloadCounter,
DepotFilesData depotFilesData,
ProtoManifest.FileData file,
ConcurrentQueue<(FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData)> networkChunkQueue)
DepotManifest.FileData file,
ConcurrentQueue<(FileStreamData, DepotManifest.FileData, DepotManifest.ChunkData)> networkChunkQueue)
{
cts.Token.ThrowIfCancellationRequested();

var depot = depotFilesData.depotDownloadInfo;
var stagingDir = depotFilesData.stagingDir;
var depotDownloadCounter = depotFilesData.depotCounter;
var oldProtoManifest = depotFilesData.previousManifest;
ProtoManifest.FileData oldManifestFile = null;
DepotManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
{
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
Expand All @@ -1030,7 +980,7 @@ private static void DownloadSteam3AsyncDepotFile(
File.Delete(fileStagingPath);
}

List<ProtoManifest.ChunkData> neededChunks;
List<DepotManifest.ChunkData> neededChunks;
var fi = new FileInfo(fileFinalPath);
var fileDidExist = fi.Exists;
if (!fileDidExist)
Expand All @@ -1048,7 +998,7 @@ private static void DownloadSteam3AsyncDepotFile(
throw new ContentDownloaderException(string.Format("Failed to allocate file {0}: {1}", fileFinalPath, ex.Message));
}

neededChunks = new List<ProtoManifest.ChunkData>(file.Chunks);
neededChunks = new List<DepotManifest.ChunkData>(file.Chunks);
}
else
{
Expand Down Expand Up @@ -1092,7 +1042,7 @@ private static void DownloadSteam3AsyncDepotFile(
fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);

var adler = Util.AdlerHash(fsOld, (int)match.OldChunk.UncompressedLength);
if (!adler.SequenceEqual(match.OldChunk.Checksum))
if (!adler.SequenceEqual(BitConverter.GetBytes(match.OldChunk.Checksum)))
{
neededChunks.Add(match.NewChunk);
}
Expand Down Expand Up @@ -1211,9 +1161,9 @@ private static async Task DownloadSteam3AsyncDepotFileChunk(
CancellationTokenSource cts,
GlobalDownloadCounter downloadCounter,
DepotFilesData depotFilesData,
ProtoManifest.FileData file,
DepotManifest.FileData file,
FileStreamData fileStreamData,
ProtoManifest.ChunkData chunk)
DepotManifest.ChunkData chunk)
{
cts.Token.ThrowIfCancellationRequested();

Expand All @@ -1222,17 +1172,8 @@ private static async Task DownloadSteam3AsyncDepotFileChunk(

var chunkID = Convert.ToHexString(chunk.ChunkID).ToLowerInvariant();

var data = new DepotManifest.ChunkData
{
ChunkID = chunk.ChunkID,
Checksum = BitConverter.ToUInt32(chunk.Checksum),
Offset = chunk.Offset,
CompressedLength = chunk.CompressedLength,
UncompressedLength = chunk.UncompressedLength
};

var written = 0;
var chunkBuffer = ArrayPool<byte>.Shared.Rent((int)data.UncompressedLength);
var chunkBuffer = ArrayPool<byte>.Shared.Rent((int)chunk.UncompressedLength);

try
{
Expand All @@ -1256,7 +1197,7 @@ private static async Task DownloadSteam3AsyncDepotFileChunk(
DebugLog.WriteLine("ContentDownloader", "Downloading chunk {0} from {1} with {2}", chunkID, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy");
written = await cdnPool.CDNClient.DownloadDepotChunkAsync(
depot.DepotId,
data,
chunk,
connection,
chunkBuffer,
depot.DepotKey,
Expand Down Expand Up @@ -1325,7 +1266,7 @@ private static async Task DownloadSteam3AsyncDepotFileChunk(
fileStreamData.fileStream = File.Open(fileFinalPath, FileMode.Open);
}

fileStreamData.fileStream.Seek((long)data.Offset, SeekOrigin.Begin);
fileStreamData.fileStream.Seek((long)chunk.Offset, SeekOrigin.Begin);
await fileStreamData.fileStream.WriteAsync(chunkBuffer.AsMemory(0, written), cts.Token);
}
finally
Expand Down Expand Up @@ -1369,44 +1310,55 @@ private static async Task DownloadSteam3AsyncDepotFileChunk(
}
}

static void DumpManifestToTextFile(DepotDownloadInfo depot, ProtoManifest manifest)
class ChunkIdComparer : IEqualityComparer<byte[]>
{
public bool Equals(byte[] x, byte[] y)
{
if (ReferenceEquals(x, y)) return true;
if (x == null || y == null) return false;
return x.SequenceEqual(y);
}

public int GetHashCode(byte[] obj)
{
ArgumentNullException.ThrowIfNull(obj);

// ChunkID is SHA-1, so we can just use the first 4 bytes
return BitConverter.ToInt32(obj, 0);
}
}

static void DumpManifestToTextFile(DepotDownloadInfo depot, DepotManifest manifest)
{
var txtManifest = Path.Combine(depot.InstallDir, $"manifest_{depot.DepotId}_{depot.ManifestId}.txt");
using var sw = new StreamWriter(txtManifest);

sw.WriteLine($"Content Manifest for Depot {depot.DepotId}");
sw.WriteLine($"Content Manifest for Depot {depot.DepotId} ");
sw.WriteLine();
sw.WriteLine($"Manifest ID / date : {depot.ManifestId} / {manifest.CreationTime}");
sw.WriteLine($"Manifest ID / date : {depot.ManifestId} / {manifest.CreationTime} ");

int numFiles = 0, numChunks = 0;
ulong uncompressedSize = 0, compressedSize = 0;
var uniqueChunks = new HashSet<byte[]>(new ChunkIdComparer());

foreach (var file in manifest.Files)
{
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
continue;

numFiles++;
numChunks += file.Chunks.Count;

foreach (var chunk in file.Chunks)
{
uncompressedSize += chunk.UncompressedLength;
compressedSize += chunk.CompressedLength;
uniqueChunks.Add(chunk.ChunkID);
}
}

sw.WriteLine($"Total number of files : {numFiles}");
sw.WriteLine($"Total number of chunks : {numChunks}");
sw.WriteLine($"Total bytes on disk : {uncompressedSize}");
sw.WriteLine($"Total bytes compressed : {compressedSize}");
sw.WriteLine($"Total number of files : {manifest.Files.Count} ");
sw.WriteLine($"Total number of chunks : {uniqueChunks.Count} ");
sw.WriteLine($"Total bytes on disk : {manifest.TotalUncompressedSize} ");
sw.WriteLine($"Total bytes compressed : {manifest.TotalCompressedSize} ");
sw.WriteLine();
sw.WriteLine();
sw.WriteLine(" Size Chunks File SHA Flags Name");

foreach (var file in manifest.Files)
{
var sha1Hash = Convert.ToHexString(file.FileHash);
sw.WriteLine($"{file.TotalSize,14} {file.Chunks.Count,6} {sha1Hash} {file.Flags,5:D} {file.FileName}");
var sha1Hash = Convert.ToHexString(file.FileHash).ToLower();
sw.WriteLine($"{file.TotalSize,14:d} {file.Chunks.Count,6:d} {sha1Hash} {(int)file.Flags,5:x} {file.FileName}");
}
}
}
Expand Down
Loading

0 comments on commit 1c97b42

Please sign in to comment.