Skip to content

Commit

Permalink
・[add] "Content-Encoding: zstd" に対応
Browse files Browse the repository at this point in the history
・[misc] 接続モニターのタイトルバーに表示されるアップロードとダウンロード速度のテキストにパッティングを追加
  • Loading branch information
amate committed Nov 18, 2024
1 parent c322dba commit aa978ad
Show file tree
Hide file tree
Showing 28 changed files with 16,341 additions and 31 deletions.
2 changes: 1 addition & 1 deletion Proxydomo/AppConst.h
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
#endif

/// アプリケーションのバージョン
#define APP_VERSION _T("2.3")
#define APP_VERSION _T("2.4")



Expand Down
4 changes: 2 additions & 2 deletions Proxydomo/ConnectionMonitor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -602,9 +602,9 @@ void CConnectionMonitorWindow::_UpdateSpeedTimer()
const int KBSeconds = bytesSeconds / 1024; // KB/s
CString text;
if (KBSeconds > 0) {
text.Format(L"%d KB/s", KBSeconds);
text.Format(L"%4d KB/s", KBSeconds);
} else {
text.Format(L"%d bytes/s", bytesSeconds);
text.Format(L"%4d bytes/s", bytesSeconds);
}
#endif
return text;
Expand Down
11 changes: 7 additions & 4 deletions Proxydomo/Proxydomo.vcxproj
Original file line number Diff line number Diff line change
Expand Up @@ -104,11 +104,11 @@
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(SolutionDir)icu\include;$(SolutionDir)Proxydomo\brotli\include;D:\Program\WTL81_12085\Include;$(SolutionDir)zlib\x64\include;C:\Programing\#lib\boost_1_73_0\lib\include\boost-1_73;$(SolutionDir)openssl-3\x64\include;$(IncludePath)</IncludePath>
<IncludePath>$(SolutionDir)icu\include;$(SolutionDir)Proxydomo\brotli\include;$(SolutionDir)Proxydomo\zstd\include;D:\Program\WTL81_12085\Include;$(SolutionDir)zlib\x64\include;C:\Programing\#lib\boost_1_73_0\lib\include\boost-1_73;$(SolutionDir)openssl-3\x64\include;$(IncludePath)</IncludePath>
<OutDir>$(SolutionDir)$(Configuration)$(Platform)\</OutDir>
<IntDir>$(Configuration)$(Platform)\</IntDir>
<TargetName>$(ProjectName)64</TargetName>
<LibraryPath>C:\Programing\#lib\boost_1_73_0\lib\lib;$(SolutionDir)icu\lib64;$(SolutionDir)zlib\x64\lib;$(SolutionDir)sqlite\Win64;$(SolutionDir)openssl-3\x64\lib;$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64);$(LibraryPath)</LibraryPath>
<LibraryPath>C:\Programing\#lib\boost_1_73_0\lib\lib;$(SolutionDir)icu\lib64;$(SolutionDir)Proxydomo\zstd\static;$(SolutionDir)zlib\x64\lib;$(SolutionDir)sqlite\Win64;$(SolutionDir)openssl-3\x64\lib;$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64);$(LibraryPath)</LibraryPath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Test|x64'">
<LinkIncremental>true</LinkIncremental>
Expand All @@ -128,8 +128,8 @@
<OutDir>$(SolutionDir)$(Configuration)$(Platform)\</OutDir>
<IntDir>$(Configuration)$(Platform)\</IntDir>
<TargetName>$(ProjectName)64</TargetName>
<LibraryPath>C:\Programing\#lib\boost_1_73_0\lib\lib;$(SolutionDir)openssl-3\x64\lib;$(SolutionDir)icu\lib64;$(SolutionDir)zlib\x64\lib;$(SolutionDir)sqlite\Win64;$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64);$(LibraryPath)</LibraryPath>
<IncludePath>$(SolutionDir)wolfssl;$(SolutionDir)icu\include;$(SolutionDir)Proxydomo\brotli\include;D:\Program\WTL81_12085\Include;$(SolutionDir)zlib\x64\include;C:\Programing\#lib\boost_1_73_0\lib\include\boost-1_73;$(SolutionDir)openssl-3\x64\include;$(IncludePath)</IncludePath>
<LibraryPath>C:\Programing\#lib\boost_1_73_0\lib\lib;$(SolutionDir)openssl-3\x64\lib;$(SolutionDir)icu\lib64;$(SolutionDir)Proxydomo\zstd\static;$(SolutionDir)zlib\x64\lib;$(SolutionDir)sqlite\Win64;$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64);$(LibraryPath)</LibraryPath>
<IncludePath>$(SolutionDir)wolfssl;$(SolutionDir)icu\include;$(SolutionDir)Proxydomo\brotli\include;$(SolutionDir)Proxydomo\zstd\include;D:\Program\WTL81_12085\Include;$(SolutionDir)zlib\x64\include;C:\Programing\#lib\boost_1_73_0\lib\include\boost-1_73;$(SolutionDir)openssl-3\x64\include;$(IncludePath)</IncludePath>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
Expand Down Expand Up @@ -852,12 +852,14 @@
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">Create</PrecompiledHeader>
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Create</PrecompiledHeader>
</ClCompile>
<ClCompile Include="stdDecompressor.cpp" />
<ClCompile Include="TextBuffer.cpp" />
<ClCompile Include="UITranslator.cpp" />
<ClCompile Include="UnitTest\Test.cpp" />
<ClCompile Include="UnitTest\Test_brotli.cpp" />
<ClCompile Include="VersionControl.cpp" />
<ClCompile Include="WinHTTPWrapper.cpp" />
<ClCompile Include="ZstdDecompressor.cpp" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="AboutDlg.h" />
Expand Down Expand Up @@ -1172,6 +1174,7 @@
<ClInclude Include="UITranslator.h" />
<ClInclude Include="VersionControl.h" />
<ClInclude Include="WinHTTPWrapper.h" />
<ClInclude Include="ZstdDecompressor.h" />
</ItemGroup>
<ItemGroup>
<ResourceCompile Include="Proxydomo.rc" />
Expand Down
57 changes: 33 additions & 24 deletions Proxydomo/Proxydomo.vcxproj.filters
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,36 @@
<ClCompile Include="BlockListDatabase.cpp">
<Filter>Header Files</Filter>
</ClCompile>
<ClCompile Include="BrotliDecompressor.cpp">
<Filter>Header Files</Filter>
</ClCompile>
<ClCompile Include="UnitTest\Test.cpp">
<Filter>Header Files\UnitTest</Filter>
</ClCompile>
<ClCompile Include="UnitTest\Test_brotli.cpp">
<Filter>Header Files\UnitTest</Filter>
</ClCompile>
<ClCompile Include="stdDecompressor.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="ZstdDecompressor.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="brotli\common\dictionary.c">
<Filter>Header Files\brotli</Filter>
</ClCompile>
<ClCompile Include="brotli\dec\bit_reader.c">
<Filter>Header Files\brotli</Filter>
</ClCompile>
<ClCompile Include="brotli\dec\decode.c">
<Filter>Header Files\brotli</Filter>
</ClCompile>
<ClCompile Include="brotli\dec\huffman.c">
<Filter>Header Files\brotli</Filter>
</ClCompile>
<ClCompile Include="brotli\dec\state.c">
<Filter>Header Files\brotli</Filter>
</ClCompile>
<ClCompile Include="brotli\enc\backward_references.c">
<Filter>Header Files\brotli</Filter>
</ClCompile>
Expand Down Expand Up @@ -255,30 +285,6 @@
<ClCompile Include="brotli\enc\utf8_util.c">
<Filter>Header Files\brotli</Filter>
</ClCompile>
<ClCompile Include="brotli\dec\bit_reader.c">
<Filter>Header Files\brotli</Filter>
</ClCompile>
<ClCompile Include="brotli\dec\decode.c">
<Filter>Header Files\brotli</Filter>
</ClCompile>
<ClCompile Include="brotli\dec\huffman.c">
<Filter>Header Files\brotli</Filter>
</ClCompile>
<ClCompile Include="brotli\dec\state.c">
<Filter>Header Files\brotli</Filter>
</ClCompile>
<ClCompile Include="brotli\common\dictionary.c">
<Filter>Header Files\brotli</Filter>
</ClCompile>
<ClCompile Include="BrotliDecompressor.cpp">
<Filter>Header Files</Filter>
</ClCompile>
<ClCompile Include="UnitTest\Test.cpp">
<Filter>Header Files\UnitTest</Filter>
</ClCompile>
<ClCompile Include="UnitTest\Test_brotli.cpp">
<Filter>Header Files\UnitTest</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="stdafx.h">
Expand Down Expand Up @@ -503,6 +509,9 @@
<ClInclude Include="Decompressor.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="ZstdDecompressor.h">
<Filter>Header Files</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<ResourceCompile Include="Proxydomo.rc">
Expand Down
6 changes: 6 additions & 0 deletions Proxydomo/RequestManager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
#include "BlockListDatabase.h"
#include "proximodo\zlibbuffer.h"
#include "BrotliDecompressor.h"
#include "ZstdDecompressor.h"

using namespace boost::filesystem;
using namespace CodeConvert;
Expand Down Expand Up @@ -1488,6 +1489,11 @@ void CRequestManager::_ProcessIn()
m_decompressor.reset(new CBrotliDecompressor());
CFilterOwner::RemoveHeader(m_filterOwner.inHeadersFiltered, L"Content-Encoding");

} else if (CUtil::noCaseContains(L"zstd", contentEncoding)) {
m_recvContentCoding = ContentEncoding::kZstd;
m_decompressor.reset(new CZstdDecompressor());
CFilterOwner::RemoveHeader(m_filterOwner.inHeadersFiltered, L"Content-Encoding");

} else if (CUtil::noCaseContains(L"identity", contentEncoding)) { // 無圧縮
m_recvContentCoding = ContentEncoding::kNone;

Expand Down
1 change: 1 addition & 0 deletions Proxydomo/RequestManager.h
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,7 @@ class CRequestManager : public IDataReceptor
kGzip,
kDeflate,
kBrotli,
kZstd,
};
ContentEncoding m_recvContentCoding;
std::unique_ptr<IDecompressor> m_decompressor;
Expand Down
51 changes: 51 additions & 0 deletions Proxydomo/ZstdDecompressor.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#include "stdafx.h"
#include "ZstdDecompressor.h"
#include <assert.h>
#include "Logger.h"
#include "CodeConvert.h"

#pragma comment(lib, "zstd.lib")

CZstdDecompressor::CZstdDecompressor()
{
m_dctx = ::ZSTD_createDCtx();
assert(m_dctx);

m_bufferOutSize = ZSTD_DStreamOutSize();
assert(m_bufferOutSize);
m_bufferOut.reset(new BYTE[m_bufferOutSize]);
}

CZstdDecompressor::~CZstdDecompressor()
{
::ZSTD_freeDCtx(m_dctx);
}

void CZstdDecompressor::feed(const std::string& data)
{
// sample: https://github.com/facebook/zstd/blob/dev/examples/streaming_decompression.c
ZSTD_inBuffer input = { static_cast<const void*>(data.data()), data.length(), 0 };
while (input.pos < input.size) {
ZSTD_outBuffer output = { static_cast<void*>(m_bufferOut.get()), m_bufferOutSize, 0 };

size_t const ret = ::ZSTD_decompressStream(m_dctx, &output, &input);
if (ZSTD_isError(ret)) {
const char* errorMessage = ZSTD_getErrorName(ret);
WARN_LOG << L"CZstdDecompressor::feed - ZSTD_decompressStream error: " << CodeConvert::UTF16fromUTF8(errorMessage);
assert(false);
return;
}
m_output.write(static_cast<const char*>(output.dst), output.pos);
}
}

void CZstdDecompressor::dump()
{
}

std::string CZstdDecompressor::read()
{
std::string out = m_output.str();
m_output.str("");
return out;
}
28 changes: 28 additions & 0 deletions Proxydomo/ZstdDecompressor.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#pragma once

#include "Decompressor.h"
#include <sstream>
#include <memory>
#include <zstd.h>

class CZstdDecompressor : public IDecompressor
{
public:
CZstdDecompressor();
~CZstdDecompressor();

// overrides
virtual void feed(const std::string& data) override;
virtual void dump() override;
virtual std::string read() override;

private:
ZSTD_DCtx* m_dctx;

size_t m_bufferOutSize;
std::unique_ptr<BYTE[]> m_bufferOut;

std::stringstream m_output;

};

63 changes: 63 additions & 0 deletions Proxydomo/zstd/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
# ZSTD Windows binary package

## The package contents

- `zstd.exe` : Command Line Utility, supporting gzip-like arguments
- `dll\libzstd.dll` : The ZSTD dynamic library (DLL)
- `dll\libzstd.lib` : The import library of the ZSTD dynamic library (DLL) for Visual C++
- `example\` : The example of usage of the ZSTD library
- `include\` : Header files required by the ZSTD library
- `static\libzstd_static.lib` : The static ZSTD library (LIB)

## Usage of Command Line Interface

Command Line Interface (CLI) supports gzip-like arguments.
By default CLI takes an input file and compresses it to an output file:

Usage: zstd [arg] [input] [output]

The full list of commands for CLI can be obtained with `-h` or `-H`. The ratio can
be improved with commands from `-3` to `-16` but higher levels also have slower
compression. CLI includes in-memory compression benchmark module with compression
levels starting from `-b` and ending with `-e` with iteration time of `-i` seconds.
CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined
into `-b1e18i1`.

## The example of usage of static and dynamic ZSTD libraries with gcc/MinGW

Use `cd example` and `make` to build `fullbench-dll` and `fullbench-lib`.
`fullbench-dll` uses a dynamic ZSTD library from the `dll` directory.
`fullbench-lib` uses a static ZSTD library from the `lib` directory.

## Using ZSTD DLL with gcc/MinGW

The header files from `include\` and the dynamic library `dll\libzstd.dll`
are required to compile a project using gcc/MinGW.
The dynamic library has to be added to linking options.
It means that if a project that uses ZSTD consists of a single `test-dll.c`
file it should be linked with `dll\libzstd.dll`. For example:

gcc $(CFLAGS) -Iinclude\ test-dll.c -o test-dll dll\libzstd.dll

The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`.

## The example of usage of static and dynamic ZSTD libraries with Visual C++

Open `example\fullbench-dll.sln` to compile `fullbench-dll` that uses a
dynamic ZSTD library from the `dll` directory. The solution works with Visual C++
2010 or newer. When one will open the solution with Visual C++ newer than 2010
then the solution will be upgraded to the current version.

## Using ZSTD DLL with Visual C++

The header files from `include\` and the import library `dll\libzstd.lib`
are required to compile a project using Visual C++.

1. The path to header files should be added to `Additional Include Directories` that can
be found in project properties `C/C++` then `General`.
2. The import library has to be added to `Additional Dependencies` that can
be found in project properties `Linker` then `Input`.
If one will provide only the name `libzstd.lib` without a full path to the library
the directory has to be added to `Linker\General\Additional Library Directories`.

The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`.
Binary file added Proxydomo/zstd/dll/libzstd.dll
Binary file not shown.
Binary file added Proxydomo/zstd/dll/libzstd.dll.a
Binary file not shown.
48 changes: 48 additions & 0 deletions Proxydomo/zstd/example/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
# ################################################################
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# You may select, at your option, one of the above-listed licenses.
# ################################################################

VOID := /dev/null
ZSTDDIR := ../include
LIBDIR := ../static
DLLDIR := ../dll

CFLAGS ?= -O3 # can select custom flags. For example : CFLAGS="-O2 -g" make
CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \
-Wdeclaration-after-statement -Wstrict-prototypes \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(MOREFLAGS)
CPPFLAGS:= -I$(ZSTDDIR) -DXXH_NAMESPACE=ZSTD_
FLAGS := $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)


# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
EXT =.exe
else
EXT =
endif

.PHONY: default fullbench-dll fullbench-lib


default: all

all: fullbench-dll fullbench-lib


fullbench-lib: fullbench.c datagen.c
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LIBDIR)/libzstd_static.lib

fullbench-dll: fullbench.c datagen.c
$(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(DLLDIR)/libzstd.dll

clean:
@$(RM) fullbench-dll$(EXT) fullbench-lib$(EXT) \
@echo Cleaning completed
Loading

0 comments on commit aa978ad

Please sign in to comment.