dedupe
MEDIUM
maintainer donnaken15
1 votes
scanned 2026-10-02 00:00:32.890515
Why flagged
The PKGBUILD downloads source files from a GitHub Gist using curl and dynamically evaluates checksums via `eval`, which could allow remote code execution if the Gist is compromised; while the downloaded files appear to be part of the project, the use of `eval` on remote content from an untrusted host increases supply-chain risk.
Triggered rules
Medium
AI review of an ambiguous pattern
llm_review
The static rules found a suspicious pattern they could not resolve, so an AI model (qwen/qwen3-235b-a22b-2507) reviewed it and judged it MEDIUM (confidence 85%): The PKGBUILD downloads source files from a GitHub Gist using curl and dynamically evaluates checksums via `eval`, which could allow remote code execution if the Gist is compromised; while the downloaded files appear to be part of the project, the use of `eval` on remote content from an untrusted host increases supply-chain risk.
-
PKGBUILD:41
eval "${_tegfunc}sums=(${sums[@]})"
PKGBUILD
1 offending line(s) highlighted
1
declare -a _file_ids=('dedupe' 'dedupe.1' 'dedupe.conf.example')
2
declare -a _inst_paths=('/usr/bin' '/usr/share/man/man1' '/usr/share/dk15')
3
srccount=${#_file_ids[@]}
4
(( myvars )) && return
5
6
# how do i force redownloading sources
7
#(( REDL != 0 )) && rm -f "${_file_ids[@]}"
8
9
# Maintainer: Wesley Kennedy <wesley@gmx.it>
10
pkgname=dedupe
11
pkgdesc='Earn back file space by hardlinking unchanging duplicate files'
12
pkgver=20260813
13
pkgrel=1
14
arch=(any)
15
license=(GPL3)
16
depends=('zsh>=5.9')
17
optdepends=('b3sum: Faster hashing function') # faulting to b2sum since it's part of coreutils
18
# compensate for the absolute STUPIDITY THAT MSYS2 IS, TRYING TO INTEGRATE
19
# PACMAN BUT AT THE SAME TIME BREAKING DEPENDENCY STUFF BECAUSE OF TRYING TO
20
# CATER TO A BUNCH OF (USELESS) ARCHITECTURES
21
b3=b3sum
22
[[ "$(uname)" = MSYS* ]] && why="$(pacman -Qqs b3sum | head -1)" && b3="$why"
23
checkdepends=(curl unzip "$b3" zstd)
24
validpgpkeys=('73311CF24AE4FF33')
25
#function btoa() { xxd -r -p <<< "$1" | base64; }
26
#function atob() { base64 -d <<< "$1" | xxd -p; }
27
source=()
28
options=()
29
_root='https://gist.githubusercontent.com/donnaken15/'
30
_gist_id='f95e8a143bb330fcf7d6268a4d6929e8'
31
_cmmt_id='7e85a0a482f87dc780b128178a7269e747c76d5e'
32
local _tegfunc=b2
33
local _digsize=512
34
declare -a sums=()
35
for (( i = 0; i < srccount; i++ )); do
36
local _src="${_root}${_gist_id}/raw/${_cmmt_id}/${_file_ids[$i]}"
37
sum="`curl -fsSL --compressed "$_src" | "${_tegfunc}sum" -b`"
38
sums+=("${sum:0:_digsize>>2}")
39
source+=("$_src")
40
done
41
eval "${_tegfunc}sums=(${sums[@]})" # forgot, is export not able to do this?
42
43
url="${_root}${_gist_id}"
44
# TODO: create dotload package (when i make other stuff)
45
source+=("${_root}2e1345bdb44cae19fa395f188fc3bef3/raw/925e9c4c261255b5d60e03668aeeaa878eee031e/dotload")
46
b2sums+=('a692dc3c77b676a236ba9ec94731318d4d2b048ba0965a9ac7ffbf19b7fd06526de424446fbbdf89f3a5ab0764da5ad8e7221bea4f082d01983c91de9272d534')
47
48
homedir='~/.config/dk15/'
49
50
install='.INSTALL'
51
>"$install" <<inst
52
post_install() {
53
echo '[1mTo create a configuration file for yourself, execute:[0m'
54
echo '[93m mkdir -p ${homedir}[0m'
55
echo '[93m cp ${_inst_paths[2]}/${_file_ids[2]} ${homedir}dedupe[0m'
56
}
57
inst
58
clean() { rm -f "$startdir/$install"; }
59
check() {
60
(( CHECK == 0 )) && {
61
plain 'Skipping test ($CHECK == 0)'
62
return 0 # HACK!!!!
63
}
64
msg2 'Run test? Requires 1.8GB. This may take a while. (Y/n)'
65
local pr; read -r pr; pr="${pr[0]:-y}"
66
[ ! "${pr,,}" = "y" ] && return 0
67
# TODO: free space check and move to /tmp if no space on device of this file, otherwise quit
68
declare -a tests=( # example archive files that would contain dupes
69
'https://sourceforge.net/projects/getgnuwin32/files/GetGnuWin32_legacy_install_archive.zip'
70
'https://github.com/twitter/twemoji/archive/7f9de7ceb8962c1b505486e23dee24282385b28b.zip' # has .ai files, before adobe was btfo'd
71
'https://github.com/jdecked/twemoji/archive/refs/heads/main.zip'
72
'https://developer.arm.com/-/media/Files/downloads/gnu-rm/10.3-2021.10/gcc-arm-none-eabi-10.3-2021.10-win32.zip' # TAKES TOO LONG ON FREAKING WSL
73
'https://releases.wikimedia.org/mediawiki/1.42/mediawiki-1.42.3.zip'
74
#'https://github.com/ungoogled-software/ungoogled-chromium-windows/releases/download/115.0.5790.171-1.1/ungoogled-chromium_115.0.5790.171-1.1_windows_x64.zip'
75
)
76
blinking=' +' # why doesn't bash have single char indexing like zsh: [x]
77
declare -a sizes=("180MB -> 397MB" "324MB -> 412MB" "8.55MB -> 12.4MB" "191MB -> 696MB" "99.3MB -> 336MB")
78
local tmpzip='./test.zip' tmpdir='./test/' tmpchk='../testchk.zst' compact=0 # not important (right now)
79
msg2 'Deleting previous temp data...'
80
(rm -rf "$tmpdir" || rm -rf $tmpdir**/*)
81
timefmt() {
82
local h m s ms time
83
(( h = ((m = (s = (ms = (time = $(("$1")))) / 1000) / 60) / 60) % 60, m %= 60, s %= 60, ms %= 1000 ))
84
# wish i could cram this in one statement
85
printf '%s%02d:%02d:%02d.%03d' "$2" $h $m $s $ms
86
}
87
now() { date '+%s%3N'; }
88
msg 'Downloading and extracting test files...'
89
start="`now`"
90
for (( i = 0; i < ${#tests[@]}; i++ )); do
91
local name="`basename "${tests[$i]}"`" out="./test/test$i" blink=1
92
local size="`curl -sI "${tests[$i]}" | sed 's/\r//g' | (grep -Ei "^Content-Length: " || echo "Content-Length: -1") | awk '{print $2}'`"
93
msg2 "$name (${sizes[$i]})"
94
curl -fSLo "$tmpzip" --progress-bar "${tests[$i]}"
95
local x=0 estc="`zipinfo -Z1 "$tmpzip" | wc -l`" width="$(head -c $((`tput cols` - 8)) < /dev/zero | tr '\0' '-')" prog lastprog=0
96
set -o pipefail
97
mkdir -p "$out"
98
unzip -od "$out" "$tmpzip" | while read -r l; do
99
(( lastprog != prog )) && {
100
printf "%s%3d%% %s%.*s%s%-*s%s"$'\r' '[97m' \
101
$(( lastprog = prog )) '[90m[[91;1m' $(( prog = (x*${#width}/estc) )) \
102
"$width" '[97m>' $((${#width}-prog)) '' '[90;22m][0m'
103
}
104
(( (prog = (x++*100/estc)) < 100 )) && echo -en "[96m${blinking:blink=! blink:1}\r" || :
105
# THIS STUPID BUILD SYSTEM THING IS SO STRICT I CAN'T DO MATH WITHOUT LAST EXPR BEING NONZERO
106
done
107
(( ! ( PIPESTATUS[0] | PIPESTATUS[1] ) )) && echo '[97mDone'
108
(( compact )) && which compact.exe >/dev/null && {
109
plain 'Compacting directory...' # FOR WHATEVER REASON
110
(pushd "$out" >/dev/null && compact.exe /c /f /s /a /i /exe:lzx '*' | tail -3)
111
}
112
rm -f "$tmpzip"
113
done
114
plain "Download time: $(timefmt "`now`-start")"
115
msg2 PERMS
116
chmod 666 -R "$tmpdir" # GnuWin32 has readonly files that get extracted
117
chmod +x dedupe
118
msg 'Readying tests...'
119
msg2 'Generating hashes of test files...'
120
set -o pipefail
121
# don't know if this can be set or unset without being typed out even in the middle of this function
122
(cd "$tmpdir" && find . -type f -print0 | xargs -0P 10 b2sum -l 128 | zstd -6c --long=26 --single-thread > "$tmpchk") # >6 not so effective in separate testing
123
# (ls -R | sha256sum) to see if list changes before and after, but it shouldn't
124
export password=alpine test_hash=b3 test_hash_length=24 test_batch_hashes=1 test_hash_workers=8
125
msg2 "Configuration: hash=$test_hash, hashlen=$test_hash_length, batch=$test_batch_hashes, hash_workers=$test_hash_workers"
126
msg 'Running test commands...'
127
msg2 'Simulation run (CRC32, scramble list)'
128
start="`now`"
129
test_simulate_mode=1 test_scramble_list=1 test_passes=1 test_hash=ck zsh -c "./dedupe '$tmpdir**/*'"
130
plain "Run time: $(timefmt "`now`-start")"
131
for i in 1 0; do
132
msg2 "Run $((2-i)), scramble=$i, sanity=$i, batch=$i, errors=$((i-1)), invalid=0"
133
start="`now`"; test_scramble_list=$i test_sanity_check=$i test_batch_hashes=$i \
134
test_hide_errored=$((1-i)) test_hide_invalid=1 zsh -c "./dedupe '$tmpdir**/*'"
135
plain "Run time: $(timefmt "`now`-start")"
136
# second run shouldn't have left over duplicates to process
137
done
138
msg2 'Run 3 (no scramble, no batching, force relinking, 1 pass)'
139
start="`now`"
140
test_hide_errored=1 test_hide_invalid=1 test_force_relink=1 test_passes=1 test_batch_hashes=0 zsh -c "./dedupe '$tmpdir**/*'"
141
plain "Run time: $(timefmt "`now`-start")"
142
msg2 'Done. Run 2 (but also pass 2 of run 1), theoretically, should have no left over files to process.'
143
set -o pipefail
144
msg 'Verifying file checksums...'
145
err='[91m'; rc='[0m'
146
# none of the file contents and where they're placed should change ever after grouping duplicates
147
echo -n "${err}" 1>&2 # ???
148
(cd "$tmpdir" && zstd -dc "$tmpchk" | b2sum --strict -cl 128 1>/dev/null) || abort=1
149
echo -n "${rc}" 1>&2
150
(( ! ( PIPESTATUS[0] | PIPESTATUS[1] | abort ) )) && msg2 'ALL OK!' || error 'Got at least one checksum mismatch.'
151
msg 'Test cleanup'
152
echo -n "${err}" 1>&2; (rm -rf "$tmpdir" || rm -rf $tmpdir**/* || exit 0); echo -n "${rc}"
153
msg2 'Test done!'
154
msg 'View the entire log with:'
155
msg2 "./logview '$pkgname-$pkgver-$pkgrel-$HOSTTYPE-check.log'"
156
(( ! abort ))
157
# print markers for -> messages so user can skip to them
158
}
159
package() {
160
local myvars=1
161
source "$startdir/PKGBUILD"
162
local man=1 # file index...
163
gzip -9c "$srcdir/${_file_ids[man]}" > "$srcdir/${_file_ids[man]}.gz"
164
_file_ids[man]="${_file_ids[man]}.gz"
165
for (( i = 0; i < srccount; i++ )); do
166
install -Dm444 "$srcdir/${_file_ids[$i]}" "$pkgdir${_inst_paths[$i]}/${_file_ids[$i]}"
167
done
168
install -Dm444 "$srcdir/dotload" "$pkgdir${_inst_paths[2]}/dotload"
169
chmod +x "${pkgdir}${_inst_paths[0]}/${_file_ids[0]}" # oops
170
}
171
Scan history
| Scanned at (UTC) | Severity | Rules |
|---|---|---|
| 2026-10-02 00:00:32 | Medium | 1 |
| 2026-10-01 00:02:06 | Medium | 1 |
| 2026-09-30 00:20:07 | Medium | 1 |
| 2026-09-29 00:07:46 | Medium | 1 |
| 2026-09-28 00:28:32 | Medium | 1 |
| 2026-09-27 00:07:07 | Medium | 1 |
| 2026-09-26 00:12:15 | Medium | 1 |
| 2026-09-25 00:03:36 | Medium | 1 |
| 2026-09-24 00:24:14 | Medium | 1 |
| 2026-09-23 00:28:13 | Medium | 1 |
| 2026-09-22 00:15:14 | Medium | 1 |
| 2026-09-21 00:26:32 | Medium | 1 |
| 2026-09-20 00:25:31 | Medium | 1 |
| 2026-09-19 00:25:36 | Medium | 1 |
| 2026-09-18 00:17:11 | Medium | 1 |
| 2026-09-17 00:27:14 | Medium | 1 |
| 2026-09-16 00:03:17 | Medium | 1 |
| 2026-09-15 00:25:31 | Medium | 1 |
| 2026-09-14 00:27:57 | Medium | 1 |
| 2026-09-13 00:19:54 | Medium | 1 |