sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-12-09 00:03:18 -05:00
Code Issues Projects Releases Wiki Activity

Adds initialization of clevel to static cdict (#3525) (#3527)

Browse Source 
- Initializes clevel in `ZSTD_CCtxParams_init`
- Adds CI workflow for msan fuzzers runs without optimization (`-O0`)
- Fixes Makefile to correctly pass on user defined `MOREFLAGS` and `FUZZER_FLAGS` in cases they have been overwritten
This commit is contained in:
Yonatan Komornik 2023-03-06 18:05:12 -08:00 committed by GitHub
parent db7d7b6974
commit 988ce61a0c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 37 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.github/workflows/dev-long-tests.yml vendored
View File

@ -186,6 +186,16 @@ jobs:
- name: MSan + Regression Test - name: MSan + Regression Test
run: make -j msanregressiontest run: make -j msanregressiontest
clang-msan-fuzz-unoptimized:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- name: clang + MSan + Fuzz Test
run: |
sudo apt-get -qqq update
sudo apt-get install clang
CC=clang MOREFLAGS="-O0" make clean msan-fuzztest
clang-msan-fuzz: clang-msan-fuzz:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:

52
Makefile
View File

@ -200,15 +200,15 @@ travis-install:
.PHONY: gcc5build gcc6build gcc7build clangbuild m32build armbuild aarch64build ppcbuild ppc64build .PHONY: gcc5build gcc6build gcc7build clangbuild m32build armbuild aarch64build ppcbuild ppc64build
gcc5build: clean gcc5build: clean
gcc-5 -v gcc-5 -v
CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror" CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)"
gcc6build: clean gcc6build: clean
gcc-6 -v gcc-6 -v
CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror" CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)"
gcc7build: clean gcc7build: clean
gcc-7 -v gcc-7 -v
CC=gcc-7 $(MAKE) all MOREFLAGS="-Werror" CC=gcc-7 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)"
clangbuild: clean clangbuild: clean
clang -v clang -v
@ -232,17 +232,17 @@ ppc64build: clean
.PHONY: armfuzz aarch64fuzz ppcfuzz ppc64fuzz .PHONY: armfuzz aarch64fuzz ppcfuzz ppc64fuzz
armfuzz: clean armfuzz: clean
CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
aarch64fuzz: clean aarch64fuzz: clean
ld -v ld -v
CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
ppcfuzz: clean ppcfuzz: clean
CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
ppc64fuzz: clean ppc64fuzz: clean
CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
.PHONY: cxxtest gcc5test gcc6test armtest aarch64test ppctest ppc64test .PHONY: cxxtest gcc5test gcc6test armtest aarch64test ppctest ppc64test
cxxtest: CXXFLAGS += -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror cxxtest: CXXFLAGS += -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
@ -251,34 +251,34 @@ cxxtest: clean
gcc5test: clean gcc5test: clean
gcc-5 -v gcc-5 -v
$(MAKE) all CC=gcc-5 MOREFLAGS="-Werror" $(MAKE) all CC=gcc-5 MOREFLAGS="-Werror $(MOREFLAGS)"
gcc6test: clean gcc6test: clean
gcc-6 -v gcc-6 -v
$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror" $(MAKE) all CC=gcc-6 MOREFLAGS="-Werror $(MOREFLAGS)"
armtest: clean armtest: clean
$(MAKE) -C $(TESTDIR) datagen # use native, faster $(MAKE) -C $(TESTDIR) datagen # use native, faster
$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
aarch64test: aarch64test:
$(MAKE) -C $(TESTDIR) datagen # use native, faster $(MAKE) -C $(TESTDIR) datagen # use native, faster
$(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
ppctest: clean ppctest: clean
$(MAKE) -C $(TESTDIR) datagen # use native, faster $(MAKE) -C $(TESTDIR) datagen # use native, faster
$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
ppc64test: clean ppc64test: clean
$(MAKE) -C $(TESTDIR) datagen # use native, faster $(MAKE) -C $(TESTDIR) datagen # use native, faster
$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
.PHONY: arm-ppc-compilation .PHONY: arm-ppc-compilation
arm-ppc-compilation: arm-ppc-compilation:
$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" $(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)"
$(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static" $(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)"
$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" $(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static $(MOREFLAGS)"
$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" $(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static $(MOREFLAGS)"
regressiontest: regressiontest:
$(MAKE) -C $(FUZZDIR) regressiontest $(MAKE) -C $(FUZZDIR) regressiontest
@ -308,31 +308,31 @@ update_regressionResults:
# run UBsan with -fsanitize-recover=pointer-overflow # run UBsan with -fsanitize-recover=pointer-overflow
# this only works with recent compilers such as gcc 8+ # this only works with recent compilers such as gcc 8+
usan: clean usan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=undefined -Werror" $(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=undefined -Werror $(MOREFLAGS)"
asan: clean asan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror" $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror $(MOREFLAGS)"
asan-%: clean asan-%: clean
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address -Werror" $(MAKE) -C $(TESTDIR) $* LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $*
msan: clean msan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer -Werror" HAVE_LZMA=0 # datagen.c fails this test for no obvious reason $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" HAVE_LZMA=0 # datagen.c fails this test for no obvious reason
msan-%: clean msan-%: clean
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer -Werror" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $* LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $*
asan32: clean asan32: clean
$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" $(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address $(MOREFLAGS)"
uasan: clean uasan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror" $(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror $(MOREFLAGS)"
uasan-%: clean uasan-%: clean
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $* LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $*
tsan-%: clean tsan-%: clean
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
.PHONY: apt-install .PHONY: apt-install
apt-install: apt-install:

1
lib/compress/zstd_compress.c
View File

@ -5453,6 +5453,7 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
params.cParams = cParams; params.cParams = cParams;
params.useRowMatchFinder = useRowMatchFinder; params.useRowMatchFinder = useRowMatchFinder;
cdict->useRowMatchFinder = useRowMatchFinder; cdict->useRowMatchFinder = useRowMatchFinder;
cdict->compressionLevel = ZSTD_NO_CLEVEL;
if (ZSTD_isError( ZSTD_initCDict_internal(cdict, if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
dict, dictSize, dict, dictSize,