python-pytorch-tokenizers

CLEAN

maintainer Smoolak 0 votes scanned 2026-10-18 00:06:55.117981

View on AUR

Triggered rules

Clean AI review downgraded a static finding llm_review

The static rules flagged this LOW, but an AI model (qwen/qwen3-235b-a22b-2507) reviewed the full PKGBUILD and judged it CLEAN (confidence 95%): The package builds from a legitimate project source on GitHub, bundles specific third-party dependencies by verified commit hashes, and applies patches to use system libraries; all actions are transparent and within normal AUR packaging practices for a Python extension with C++ components.

1 higher static finding superseded - not the current verdict (shown for transparency)

Low Few votes, recently uploaded zero_votes_recent

Uploaded within the last 14 days with 2 or fewer community votes — little peer review so far.

PKGBUILD

      1# Maintainer: Smoolak <smoolak@gmail.com>
    
      2
    
      3pkgname=python-pytorch-tokenizers
    
      4_pkgname=tokenizers
    
      5pkgver=1.5.0
    
      6pkgrel=1
    
      7pkgdesc='Common C++ and Python tokenizers for the PyTorch ecosystem'
    
      8arch=('x86_64' 'aarch64')
    
      9url='https://github.com/meta-pytorch/tokenizers'
    
      10license=('BSD-3-Clause')
    
      11options=('!debug')
    
      12depends=(
    
      13  'abseil-cpp'
    
      14  'glibc'
    
      15  'libgcc'
    
      16  'libstdc++'
    
      17  'python'
    
      18  'python-sentencepiece'
    
      19  'python-tiktoken'
    
      20  'python-tokenizers'
    
      21  're2'
    
      22  'sentencepiece'
    
      23)
    
      24makedepends=(
    
      25  'cmake'
    
      26  'ninja'
    
      27  'pkgconf'
    
      28  'pybind11'
    
      29  'python-build'
    
      30  'python-installer'
    
      31  'python-setuptools'
    
      32  'python-wheel'
    
      33)
    
      34checkdepends=(
    
      35  'gtest'
    
      36  'python-pytest'
    
      37)
    
      38optdepends=(
    
      39  'python-mistral-common: reference interoperability checks for Tekken tokenizers'
    
      40)
    
      41_json_commit=e6cafa573aac6ed9227f752a5371c0b3f436307d
    
      42_pcre2_commit=2e03e323339ab692640626f02f8d8d6f95bff9c6
    
      43_sljit_commit=98802d939be3f0ccc2c538d611ec4c728564e8cf
    
      44source=(
    
      45  "$pkgname-$pkgver.tar.gz::$url/archive/refs/tags/v$pkgver.tar.gz"
    
      46  "json-$_json_commit.tar.gz::https://github.com/nlohmann/json/archive/$_json_commit.tar.gz"
    
      47  "pcre2-$_pcre2_commit.tar.gz::https://github.com/PCRE2Project/pcre2/archive/$_pcre2_commit.tar.gz"
    
      48  "sljit-$_sljit_commit.tar.gz::https://github.com/zherczeg/sljit/archive/$_sljit_commit.tar.gz"
    
      49  'use-system-pybind11-and-gtest.patch'
    
      50  'Use-declared-system-build-dependencies.patch'
    
      51  'use-system-sentencepiece.patch'
    
      52  'test-system-libraries.patch'
    
      53)
    
      54sha256sums=(
    
      55  '6ecd53de4d595f8072e66bfd58e6f4728237ac03bceee190dd07f80771d29837'
    
      56  '0b0fdb7f3065b1bd2c34a6faf369f8e4a19b60832298bab498456dd0a832a730'
    
      57  'd2bab343b55525fdfe322db020720bd84afc8233d5d428165eec4135aad85761'
    
      58  '5e91c146ab328fd915a03198d6b88a74a696e4ac17abedee34a90bea516e708a'
    
      59  'de31ec275842ff39a2405a6d3d6461094f21baab7935c20945cdfac6b18bd54b'
    
      60  'b419022b3e1e0b0632b2ae0107125efe36694660c595a072d5c1f37f3796cf57'
    
      61  'f20f96ce2964b22414555d0ad105495cea3bfce6c8f8f2c1192fd959d09510f2'
    
      62  'a6635d00306c4bfa25407dd8770cb95f05145d0291923798b24f38b666454fd2'
    
      63)
    
      64
    
      65prepare() {
    
      66  cd "$_pkgname-$pkgver"
    
      67
    
      68  rm -rf \
    
      69    third-party/json \
    
      70    third-party/pcre2
    
      71  mkdir -p third-party
    
      72  cp -a "$srcdir/json-$_json_commit" third-party/json
    
      73  cp -a "$srcdir/pcre2-$_pcre2_commit" third-party/pcre2
    
      74  rm -rf third-party/pcre2/deps/sljit
    
      75  mkdir -p third-party/pcre2/deps
    
      76  cp -a "$srcdir/sljit-$_sljit_commit" third-party/pcre2/deps/sljit
    
      77
    
      78  patch -Np1 --fuzz=0 -i "$srcdir/use-system-pybind11-and-gtest.patch"
    
      79  patch -Np1 --fuzz=0 -i "$srcdir/Use-declared-system-build-dependencies.patch"
    
      80  patch -Np1 --fuzz=0 -i "$srcdir/use-system-sentencepiece.patch"
    
      81  patch -Np1 --fuzz=0 -i "$srcdir/test-system-libraries.patch"
    
      82}
    
      83
    
      84build() {
    
      85  cd "$_pkgname-$pkgver"
    
      86  export CMAKE_GENERATOR=Ninja
    
      87  python -m build --wheel --no-isolation
    
      88}
    
      89
    
      90check() {
    
      91  local _source_dir="$srcdir/$_pkgname-$pkgver"
    
      92  local _cpp_build="$srcdir/cpp-tests"
    
      93  local _test_root="$srcdir/test-work"
    
      94  local _test_venv="$srcdir/test-venv"
    
      95
    
      96  rm -rf "$_cpp_build" "$_test_root" "$_test_venv"
    
      97  cmake -S "$_source_dir/test" -B "$_cpp_build" -G Ninja \
    
      98    -DCMAKE_BUILD_TYPE=Release
    
      99  cmake --build "$_cpp_build"
    
      100  ctest --test-dir "$_cpp_build" --output-on-failure
    
      101
    
      102  mkdir -p "$_test_root"
    
      103  cp -a "$_source_dir/test" "$_test_root/test"
    
      104  python -m venv --system-site-packages "$_test_venv"
    
      105  "$_test_venv/bin/python" -m installer "$_source_dir"/dist/*.whl
    
      106  cd "$_test_root"
    
      107  PYTHONDONTWRITEBYTECODE=1 "$_test_venv/bin/python" -m pytest -vv \
    
      108    test/test_python_bindings.py \
    
      109    test/test_tiktoken.py \
    
      110    test/test_tekken_python.py
    
      111}
    
      112
    
      113package() {
    
      114  cd "$_pkgname-$pkgver"
    
      115  python -m installer --destdir="$pkgdir" dist/*.whl
    
      116  install -Dm644 LICENSE "$pkgdir/usr/share/licenses/$pkgname/LICENSE"
    
      117}
    
      118

Scan history

Scanned at (UTC)	Severity	Rules
2026-10-18 00:06:55	Clean	2
2026-10-17 23:20:05	Low	1