Merge branch 'master' into fix/soft-links-should-remain-the-same

build-fuzz-corpus-multiple-restarts.sh

@@ -2,19 +2,59 @@
 set -euxo pipefail
 ncpus=$(nproc || getconf NPROCESSORS_ONLN)
 ncpus=$(( ncpus / ( 1 + $(cat /sys/devices/system/cpu/smt/active))))
-RESTARTS=10
-mv "fuzz/corpus/fuzz_$1" "fuzz/corpus/fuzz_$1_pre_fresh_blood" || true
-for i in $(seq 1 $RESTARTS); do
-  echo "RESTART ${i}"
-  mkdir "fuzz/corpus/fuzz_$1"
+NORMAL_RESTARTS=5
+rm -rf "fuzz/corpus/fuzz_$1_pre_fresh_blood" || true
+mkdir "fuzz/corpus/fuzz_$1_pre_fresh_blood"
+find "fuzz/corpus/fuzz_$1" -type f -exec mv '{}' "fuzz/corpus/fuzz_$1_pre_fresh_blood" ';' || true
+for i in $(seq 1 $NORMAL_RESTARTS); do
+  find "fuzz/corpus/fuzz_$1_restart_${i}" -type f -exec mv '{}' "fuzz/corpus/fuzz_$1_pre_fresh_blood" ';' || true
+  rm -rf "fuzz/corpus/fuzz_$1_restart_${i}" || true
+  echo "$(date): RESTART ${i}"
+  mkdir "fuzz/corpus/fuzz_$1" || true
   cargo fuzz run --all-features "fuzz_$1" "fuzz/corpus/fuzz_$1" -- \
     -dict=fuzz/fuzz.dict -max_len="$2" -fork="$ncpus" \
     -max_total_time=5100 -runs=100000000
   mv "fuzz/corpus/fuzz_$1" "fuzz/corpus/fuzz_$1_restart_${i}"
+  mkdir "fuzz/corpus/fuzz_$1"
 done
+
+find "fuzz/corpus/fuzz_$1_restart_dictionaryless" -type f -exec mv '{}' "fuzz/corpus/fuzz_$1_pre_fresh_blood" ';' || true
+rm -rf "fuzz/corpus/fuzz_$1_restart_dictionaryless" || true
+echo "$(date): DICTIONARY-LESS RESTART"
+cargo fuzz run --all-features "fuzz_$1" "fuzz/corpus/fuzz_$1" -- \
+  -max_len="$2" -fork="$ncpus" -max_total_time=5100 -runs=100000000
+mv "fuzz/corpus/fuzz_$1" "fuzz/corpus/fuzz_$1_restart_dictionaryless"
 mkdir "fuzz/corpus/fuzz_$1"
-for i in $(seq 1 $RESTARTS); do
-  mv "fuzz/corpus/fuzz_$1_restart_${i}"/* "fuzz/corpus/fuzz_$1"
-  rmdir "fuzz/corpus/fuzz_$1_restart_${i}"
+
+find "fuzz/corpus/fuzz_$1_restart_dictionaryless_012byte" -type f -exec mv '{}' "fuzz/corpus/fuzz_$1_pre_fresh_blood" ';' || true
+rm -rf "fuzz/corpus/fuzz_$1_restart_dictionaryless_012byte" || true
+echo "$(date): DICTIONARY-LESS RESTART WITH 0-2 BYTE CORPUS"
+tar -xvzf "fuzz/012byte.tar.gz" -C "fuzz/corpus/fuzz_$1"
+cargo fuzz run --all-features "fuzz_$1" "fuzz/corpus/fuzz_$1" -- \
+  -max_len="$2" -fork="$ncpus" -max_total_time=5100 -runs=100000000
+mv "fuzz/corpus/fuzz_$1" "fuzz/corpus/fuzz_$1_restart_dictionaryless_012byte"
+mkdir "fuzz/corpus/fuzz_$1"
+
+find "fuzz/corpus/fuzz_$1_restart_012byte" -type f -exec mv '{}' "fuzz/corpus/fuzz_$1_pre_fresh_blood" ';' || true
+rm -rf "fuzz/corpus/fuzz_$1_restart_012byte" || true
+echo "$(date): RESTART WITH DICTIONARY AND 0-2 BYTE CORPUS"
+tar -xvzf "fuzz/012byte.tar.gz" -C "fuzz/corpus/fuzz_$1"
+cargo fuzz run --all-features "fuzz_$1" "fuzz/corpus/fuzz_$1" -- \
+  -dict=fuzz/fuzz.dict -max_len="$2" -fork="$ncpus" -max_total_time=5100 -runs=100000000
+
+echo "$(date): MERGING CORPORA"
+for i in $(seq 1 $NORMAL_RESTARTS); do
+  find "fuzz/corpus/fuzz_$1_restart_${i}" -type f -exec mv '{}' "fuzz/corpus/fuzz_$1" ';'
+  rm -rf "fuzz/corpus/fuzz_$1_restart_${i}"
 done
-./fuzz-until-converged.sh $1 $2
+SPECIAL_RESTARTS=("dictionaryless_012byte" "dictionaryless")
+for i in "${SPECIAL_RESTARTS[@]}"; do
+  find "fuzz/corpus/fuzz_$1_restart_${i}" -type f -exec mv '{}' "fuzz/corpus/fuzz_$1" ';'
+  rm -rf "fuzz/corpus/fuzz_$1_restart_${i}"
+done
+echo "$(date): RUNNING WITH MERGED CORPUS"
+cargo fuzz run --all-features "fuzz_$1" "fuzz/corpus/fuzz_$1" -- \
+  -dict=fuzz/fuzz.dict -max_len="$2" -fork="$ncpus" \
+  -max_total_time=1800 -runs=25000000 -rss_limit_mb=8192 -timeout=30
+./recursive-fuzz-cmin.sh "$1" "$2"
+echo "$(date): DONE BUILDING FUZZ CORPUS AT SIZE $2"

build-fuzz-corpus-multiple-sizes.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -euxo pipefail
+mkdir "fuzz/corpus/fuzz_$1_recombination_sources" || true
+
+# Ensure the 0-byte, 1-byte and 2-byte strings won't gain duplicates during recombination
+find "fuzz/corpus/fuzz_$1_recombination_sources" -type f -size -3c -delete
+
+for size in "${@:2}"; do
+  echo "$(date): STARTING ON SIZE $size"
+  rm -rf "fuzz/corpus/fuzz_$1_pre_fresh_blood" || true
+  find "fuzz/corpus/fuzz_$1" -type f -exec mv '{}' "fuzz/corpus/fuzz_$1_recombination_sources" ';' || true
+  ./build-fuzz-corpus-multiple-restarts.sh "$1" "$size"
+  find "fuzz/corpus/fuzz_$1_recombination_sources" -type f -size "-$((size + 1))c" -exec mv '{}' "fuzz/corpus/fuzz_$1" ';'
+  ./fuzz-until-converged.sh "$1" "$size"
+done
+echo "$(date): FINISHED"

fuzz-until-converged.sh

@@ -9,7 +9,7 @@ while [[ $iters_without_improvement -lt $MAX_ITERS_WITHOUT_IMPROVEMENT ]]; do
   cp -r "fuzz/corpus/fuzz_$1" "fuzz/corpus/fuzz_$1_old"
   cargo fuzz run --all-features "fuzz_$1" "fuzz/corpus/fuzz_$1" -- \
     -dict=fuzz/fuzz.dict -max_len="$2" -fork="$ncpus" \
-    -max_total_time=1800 -runs=20000000 -rss_limit_mb=8192 -timeout=30
+    -max_total_time=1800 -runs=25000000 -rss_limit_mb=8192 -timeout=30
   ./recursive-fuzz-cmin.sh "$1" "$2"
   if diff "fuzz/corpus/fuzz_$1" "fuzz/corpus/fuzz_$1_old"; then
     iters_without_improvement=$(( iters_without_improvement + 1 ))

fuzz/corpus/fuzz_write/000dc1ed9aa712aed7be20f742167c68d57154ea

@@ -1,2 +0,0 @@
-<EFBFBD><EFBFBD><EFBFBD><EFBFBD><02><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>PK./
-,

fuzz/corpus/fuzz_write/0011743621118ab6c5278ffbb8fd14bddd8369ee

@@ -0,0 +1 @@
+
e†Úÿucœ/.ˆuDDD
!DDDDDDDD

fuzz/corpus/fuzz_write/004ea17d4b08b22f3f15a67f4e42935a2f0021be

@@ -0,0 +1 @@
+ÉßõëûÿU	K§Po.;/= -¹Ý

fuzz/corpus/fuzz_write/014a09bf446a6cf3c008d493f3f0b5b73d0720b8

@@ -1 +0,0 @@
-;<3B><><EFBFBD>C//<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>;<3B><><EFBFBD><EFBFBD><EFBFBD>

fuzz/corpus/fuzz_write/01b057838deb08950a83c2b6b5b5a1cfe29ab9eb

@@ -0,0 +1,5 @@
+/.//
+u
+/ccK#
+ccK#
+PP///cc

fuzz/corpus/fuzz_write/01e8411a0381b9ae5227f12d8e651aececac3117

@@ -0,0 +1 @@
+/<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>&<26><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>UT<55><54><EFBFBD>Є<EFBFBD><D084><EFBFBD><EFBFBD><EFBFBD>*/KKP

fuzz/corpus/fuzz_write/026cabc695756d69a3dbe1e08c8dae3d7723db19

@@ -0,0 +1 @@
+?<3F>PL<06>[<5B>
P<><50><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>PK<50><4B><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>P2<>222

fuzz/corpus/fuzz_write/0273f25c8edf5aaee565b03bf1cb2de141fca9dc

@@ -0,0 +1 @@
+/../#
;
<01>;
<01>F<EFBFBD><46>UPK/TA.Y

fuzz/corpus/fuzz_write/028cc6b91630876c879cf2922427c34e7cd90dcb

@@ -1 +0,0 @@
-<0B><><EFBFBD>I防

fuzz/corpus/fuzz_write/02c867fb8c0740ab00020d4583e810aa0230441e

@@ -0,0 +1 @@
+<EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Yqc<71><63>c<EFBFBD><63><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>c.//