diff --git a/.gitattributes b/.gitattributes index 35861e1c..dd49f639 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1 @@ -include/** linguist-vendored +include/yyjson/* linguist-vendored diff --git a/.github/actions/manylinux/action.yaml b/.github/actions/manylinux/action.yaml new file mode 100644 index 00000000..d6688f20 --- /dev/null +++ b/.github/actions/manylinux/action.yaml @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2024-2026) + +name: manylinux + +inputs: + arch: + required: true + interpreter: + required: true + features: + required: true + compatibility: + required: true + packages: + required: true + +runs: + using: "composite" + steps: + + - name: Build and test + shell: bash + run: | + set -eou pipefail + + mkdir dist + + export PYTHON="${{ inputs.interpreter }}" + if [[ "${PYTHON}" == *t ]]; then + export PYTHON_PACKAGE="$(echo ${PYTHON} | sed 's/.$//')-freethreading" + else + export PYTHON_PACKAGE="${PYTHON}" + fi + + export TARGET="${{ inputs.arch }}-unknown-linux-gnu" + + ./script/install-fedora "${{ inputs.packages }}" + + source "${VENV}/bin/activate" + + maturin build \ + --release \ + --strip \ + --features="${{ inputs.features }}" \ + --compatibility="${{ inputs.compatibility }}" \ + --interpreter="${PYTHON}" \ + --target="${TARGET}" + + uv pip install ${CARGO_TARGET_DIR}/wheels/orjson*.whl + + cp ${CARGO_TARGET_DIR}/wheels/orjson*.whl dist diff --git a/.github/workflows/artifact.yaml b/.github/workflows/artifact.yaml new file mode 100644 index 00000000..2cf4f435 --- /dev/null +++ b/.github/workflows/artifact.yaml @@ -0,0 +1,779 @@ +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2022-2026), messense (2022), Dominic LoBue (2022) + +name: artifact +on: push +env: + FORCE_COLOR: "1" + ORJSON_BUILD_FREETHREADED: "1" + PIP_DISABLE_PIP_VERSION_CHECK: "1" + RUST_TOOLCHAIN: "nightly-2026-05-01" + RUST_TOOLCHAIN_STABLE: "1.95" + UNSAFE_PYO3_BUILD_FREE_THREADED: "1" + UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" + UV_LINK_MODE: "copy" +jobs: + + sdist: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + strategy: + fail-fast: false + steps: + + - uses: actions/setup-python@v6 + with: + python-version: "3.14" + + - name: rustup stable + run: | + curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain "${RUST_TOOLCHAIN_STABLE}" -y + rustup default "${RUST_TOOLCHAIN_STABLE}" + + - uses: actions/checkout@v6 + + - name: Cargo.toml and pyproject.toml version must match + run: ./script/check-version + + - run: python3 -m pip install --user --upgrade pip "maturin>=1.13.1,<2" wheel + + - name: Vendor dependencies + run: | + maturin build + cargo fetch + mkdir .cargo + cp ci/sdist.toml .cargo/config.toml + cargo vendor include/cargo --versioned-dirs + + - run: maturin sdist --out=dist + + - run: python3 -m pip install --user dist/orjson*.tar.gz + env: + CARGO_NET_OFFLINE: "true" + + - run: python3 -m pip install --user -r test/requirements.txt -r integration/requirements.txt mypy + + - run: pytest -v test + env: + PYTHONMALLOC: "debug" + + - run: ./integration/run thread + - run: ./integration/run http + - run: ./integration/run init + - run: ./integration/run typestubs + + - name: Store sdist + uses: actions/upload-artifact@v7 + with: + name: orjson_sdist + path: dist + overwrite: true + retention-days: 1 + if-no-files-found: "error" + compression-level: 0 + + manylinux: + name: "manylinux_${{ matrix.arch.arch }}_${{ matrix.python.interpreter }}" + runs-on: "${{ matrix.arch.runner }}" + container: + image: fedora:rawhide + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python: [ + { interpreter: 'python3.15', compatibility: "manylinux_2_39", publish: false }, + { interpreter: 'python3.15t', compatibility: "manylinux_2_39", publish: false }, + { interpreter: 'python3.14', compatibility: "manylinux_2_17", publish: true }, + { interpreter: 'python3.13', compatibility: "manylinux_2_17", publish: true }, + { interpreter: 'python3.12', compatibility: "manylinux_2_17", publish: true }, + { interpreter: 'python3.11', compatibility: "manylinux_2_17", publish: true }, + { interpreter: 'python3.10', compatibility: "manylinux_2_17", publish: true }, + ] + arch: [ + { runner: "ubuntu-24.04", arch: "x86_64", features: "avx512,optimize,no_panic", }, + { runner: "ubuntu-24.04-arm", arch: "aarch64", features: "generic_simd,optimize,no_panic" }, + ] + env: + CARGO_TARGET_DIR: "/tmp/orjson" + CC: "clang" + CFLAGS: "-O2 -fstrict-aliasing -fno-plt -emit-llvm" + LDFLAGS: "-fuse-ld=lld -Wl,-plugin-opt=also-emit-llvm -Wl,--as-needed -Wl,-zrelro,-znow" + PATH: "/__w/orjson/orjson/.venv/bin:/github/home/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + RUSTFLAGS: "-Z unstable-options -C panic=immediate-abort -C linker=clang -C link-arg=-fuse-ld=lld -C linker-plugin-lto -C link-arg=-Wl,-zrelro,-znow -Z mir-opt-level=4 -Z threads=4 -D warnings" + VENV: ".venv" + steps: + + - name: CPU info + run: cat /proc/cpuinfo + + - run: dnf install --setopt=install_weak_deps=false -y git-core + + - uses: actions/checkout@v6 + + - name: Build and test + uses: ./.github/actions/manylinux + with: + arch: "${{ matrix.arch.arch }}" + interpreter: "${{ matrix.python.interpreter }}" + features: "${{ matrix.arch.features }}" + compatibility: "${{ matrix.python.compatibility }}" + packages: "" + + - run: pytest -v test + env: + PYTHONMALLOC: "debug" + + - run: ./integration/run thread + env: + PYTHONMALLOC: "debug" + + - run: ./integration/run http + env: + PYTHONMALLOC: "debug" + + - run: ./integration/run init + env: + PYTHONMALLOC: "debug" + + - name: Store wheels + if: matrix.python.publish == true + uses: actions/upload-artifact@v7 + with: + name: "orjson_manylinux_${{ matrix.arch.arch }}_${{ matrix.python.interpreter }}_${{ matrix.python.compatibility }}" + path: dist + overwrite: true + retention-days: 1 + + - name: Debug + env: + CARGO_TARGET_DIR: "/tmp/orjson" + PYTHON: "${{ matrix.python.interpreter }}" + TARGET: "${{ matrix.arch.arch }}-unknown-linux-gnu" + run: | + export PATH="$PWD/.venv:$HOME/.cargo/bin:$PATH" + source .venv/bin/activate + script/debug + + manylinux_cross: + name: "manylinux_${{ matrix.target.arch }}_${{ matrix.python.interpreter }}" + runs-on: ${{ matrix.target.runner }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python: [ + { interpreter: 'python3.14', abi: 'cp314-cp314', manylinux: 'manylinux_2_17', maturin-version: 'v1.11.5', publish: true }, + { interpreter: 'python3.13', abi: 'cp313-cp313', manylinux: 'manylinux_2_17', maturin-version: 'v1.13.1', publish: true }, + { interpreter: 'python3.12', abi: 'cp312-cp312', manylinux: 'manylinux_2_17', maturin-version: 'v1.13.1', publish: true }, + { interpreter: 'python3.11', abi: 'cp311-cp311', manylinux: 'manylinux_2_17', maturin-version: 'v1.13.1', publish: true }, + { interpreter: 'python3.10', abi: 'cp310-cp310', manylinux: 'manylinux_2_17', maturin-version: 'v1.13.1', publish: true }, + ] + target: [ + { + arch: 'i686', + cflags: '-Os -fstrict-aliasing', + features: 'no_panic,optimize', + image: 'quay.io/pypa/manylinux_2_28_i686:latest', + runner: "ubuntu-24.04", + rustflags: '-Z unstable-options -C panic=immediate-abort -Z mir-opt-level=4 -D warnings', + target: 'i686-unknown-linux-gnu', + }, + { + arch: 'armv7', + cflags: '-Os -fstrict-aliasing', + features: 'no_panic,optimize', + image: 'ghcr.io/rust-cross/manylinux_2_28-cross:armv7', + runner: "ubuntu-24.04-arm", + rustflags: '-Z unstable-options -C panic=immediate-abort -Z mir-opt-level=4 -D warnings -C opt-level=s', + target: 'armv7-unknown-linux-gnueabihf', + }, + { + arch: 'ppc64le', + cflags: '-Os -fstrict-aliasing', + features: 'generic_simd,no_panic,optimize', + image: 'ghcr.io/rust-cross/manylinux_2_28-cross:ppc64le', + runner: "ubuntu-24.04", + rustflags: '-Z unstable-options -C panic=immediate-abort -Z mir-opt-level=4 -D warnings', + target: 'powerpc64le-unknown-linux-gnu', + }, + { + arch: 's390x', + cflags: '-Os -fstrict-aliasing -march=z10', + features: 'no_panic,optimize', + image: 'ghcr.io/rust-cross/manylinux_2_28-cross:s390x', + runner: "ubuntu-24.04", + rustflags: '-Z unstable-options -C panic=immediate-abort -Z mir-opt-level=4 -D warnings -C target-cpu=z10', + target: 's390x-unknown-linux-gnu', + }, + ] + steps: + - uses: actions/checkout@v6 + + - name: build-std + run: | + mkdir .cargo + cp ci/config.toml .cargo/config.toml + + - name: Build + uses: PyO3/maturin-action@v1 + env: + PYO3_CROSS_LIB_DIR: "/opt/python/${{ matrix.python.abi }}" + CFLAGS: "${{ matrix.target.cflags }}" + LDFLAGS: "-Wl,--as-needed" + RUSTFLAGS: "${{ matrix.target.rustflags }}" + with: + args: --release --strip --out=dist --features=${{ matrix.target.features }} -i ${{ matrix.python.interpreter }} + container: "${{ matrix.target.image }}" + manylinux: "${{ matrix.python.manylinux }}" + maturin-version: "${{ matrix.python.maturin-version }}" + rust-toolchain: "${{ env.RUST_TOOLCHAIN }}" + rustup-components: rust-src + target: "${{ matrix.target.target }}" + + - name: Store wheels + if: matrix.python.publish == true + uses: actions/upload-artifact@v7 + with: + name: "orjson_manylinux_${{ matrix.target.arch }}_${{ matrix.python.interpreter }}" + path: dist + overwrite: true + retention-days: 1 + if-no-files-found: "error" + compression-level: 0 + + - name: setup-qemu-container + if: "matrix.target.arch == 's390x' || matrix.target.arch == 'ppc64le'" + uses: sandervocke/setup-qemu-container@v1 + with: + container: registry.fedoraproject.org/fedora:43 + arch: ${{ matrix.target.arch }} + podman_args: "-v .:/orjson -v /tmp:/tmp --workdir /orjson" + + - name: setup-shell-wrapper + uses: sandervocke/setup-shell-wrapper@v1 + + - name: Emulated Test + if: "matrix.target.arch == 's390x' || matrix.target.arch == 'ppc64le'" + shell: wrap-shell {0} + env: + WRAP_SHELL: run-in-container.sh + run: | + set -eou pipefail + + dnf install --setopt=install_weak_deps=false -y ${{ matrix.python.interpreter }} python3-uv + + uv venv --python ${{ matrix.python.interpreter }} + source .venv/bin/activate + + uv pip install -r test/requirements.txt + uv pip install dist/orjson*.whl + + pytest -v test + + musllinux_amd64: + name: "musllinux_amd64_python${{ matrix.python.version }}" + runs-on: ubuntu-24.04 + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python: [ + { version: '3.14', pytest: '1', publish: true }, + { version: '3.13', pytest: '1', publish: true }, + { version: '3.12', pytest: '1', publish: true }, + { version: '3.11', pytest: '0', publish: true }, + { version: '3.10', pytest: '0', publish: true }, + ] + platform: + - target: x86_64-unknown-linux-musl + arch: x86_64 + platform: linux/amd64 + features: avx512,no_panic,optimize,unwind + - target: i686-unknown-linux-musl + arch: i686 + platform: linux/386 + features: no_panic,optimize,unwind + steps: + - uses: actions/checkout@v6 + + - name: build-std + run: | + mkdir .cargo + cp ci/config.toml .cargo/config.toml + + - name: Build + uses: PyO3/maturin-action@v1 + env: + CC: "gcc" + CFLAGS: "-O2" + LDFLAGS: "-Wl,--as-needed" + RUSTFLAGS: "-Z unstable-options -C panic=immediate-abort -Z mir-opt-level=4 -Z threads=2 -D warnings -C target-feature=-crt-static" + with: + rust-toolchain: "${{ env.RUST_TOOLCHAIN }}" + rustup-components: rust-src + target: "${{ matrix.platform.target }}" + manylinux: musllinux_1_2 + args: --release --strip --out=dist --features=${{ matrix.platform.features }} -i python${{ matrix.python.version }} + + - name: Test + uses: addnab/docker-run-action@v3 + with: + image: "quay.io/pypa/musllinux_1_2_${{ matrix.platform.arch }}:2026.03.27-1" + options: -v ${{ github.workspace }}:/io -w /io + run: | + apk add tzdata + sed -i '/^psutil/d' test/requirements.txt # missing 3.11, 3.12 wheels + sed -i '/^numpy/d' test/requirements.txt + + python${{ matrix.python.version }} -m venv venv + venv/bin/pip install -U pip wheel + venv/bin/pip install orjson --no-index --find-links dist/ --force-reinstall + + # segfault on starting pytest after January 2025 on 3.11 and older; artifact works fine + if [ ${{ matrix.python.pytest }} == '1' ]; then + venv/bin/pip install -r test/requirements.txt + PYTHONMALLOC="debug" venv/bin/python -m pytest -v test + fi + + - name: Store wheels + if: matrix.python.publish == true + uses: actions/upload-artifact@v7 + with: + name: orjson_musllinux_${{ matrix.platform.arch }}_${{ matrix.python.version }} + path: dist + overwrite: true + retention-days: 1 + if-no-files-found: "error" + compression-level: 0 + + musllinux_aarch64: + name: "musllinux_aarch64_python${{ matrix.python.version }}" + runs-on: ubuntu-24.04-arm + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python: [ + { version: '3.14', publish: true }, + { version: '3.13', publish: true }, + { version: '3.12', publish: true }, + { version: '3.11', publish: true }, + { version: '3.10', publish: true }, + ] + platform: + - target: aarch64-unknown-linux-musl + arch: aarch64 + platform: linux/arm64 + features: generic_simd,no_panic,optimize,unwind + - target: armv7-unknown-linux-musleabihf + arch: armv7l + platform: linux/arm/v7 + features: no_panic,optimize + steps: + - uses: actions/checkout@v6 + + - name: build-std + run: | + mkdir .cargo + cp ci/config.toml .cargo/config.toml + + - name: Build + uses: PyO3/maturin-action@v1 + env: + CC: "gcc" + CFLAGS: "-O2" + LDFLAGS: "-Wl,--as-needed" + RUSTFLAGS: "-Z unstable-options -C panic=immediate-abort -Z mir-opt-level=4 -Z threads=2 -D warnings -C target-feature=-crt-static" + with: + rust-toolchain: "${{ env.RUST_TOOLCHAIN }}" + rustup-components: rust-src + target: "${{ matrix.platform.target }}" + manylinux: musllinux_1_2 + args: --release --strip --out=dist --features=${{ matrix.platform.features }} -i python${{ matrix.python.version }} + + - name: Test + uses: addnab/docker-run-action@v3 + with: + image: "quay.io/pypa/musllinux_1_2_${{ matrix.platform.arch }}:2026.03.27-1" + options: -v ${{ github.workspace }}:/io -w /io + run: | + apk add tzdata + sed -i '/^psutil/d' test/requirements.txt # missing 3.11, 3.12 wheels + sed -i '/^numpy/d' test/requirements.txt + + python${{ matrix.python.version }} -m venv venv + venv/bin/pip install -U pip wheel + venv/bin/pip install -r test/requirements.txt + venv/bin/pip install orjson --no-index --find-links dist/ --force-reinstall + export PYTHONMALLOC="debug" + venv/bin/python -m pytest -v test + + - name: Store wheels + if: matrix.python.publish == true + uses: actions/upload-artifact@v7 + with: + name: orjson_musllinux_${{ matrix.platform.arch }}_${{ matrix.python.version }} + path: dist + overwrite: true + retention-days: 1 + if-no-files-found: "error" + compression-level: 0 + + macos_aarch64: + name: "macos_aarch64_python${{ matrix.python.version }}" + runs-on: macos-26 + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python: [ + { version: '3.14', macosx_target: "15.0", publish: true }, + { version: '3.13', macosx_target: "15.0", publish: true }, + { version: '3.12', macosx_target: "15.0", publish: true }, + { version: '3.11', macosx_target: "15.0", publish: true }, + ] + env: + CC: "clang" + LDFLAGS: "-Wl,--as-needed" + CFLAGS: "-O2 -fstrict-aliasing -fno-plt -mcpu=apple-m1 -mtune=generic" + RUSTFLAGS: "-Z unstable-options -C panic=immediate-abort -Z mir-opt-level=4 -Z threads=3 -D warnings" + PATH: "/Users/runner/work/orjson/orjson/.venv/bin:/Users/runner/.cargo/bin:/usr/local/opt/curl/bin:/usr/local/bin:/usr/local/sbin:/Users/runner/bin:/Library/Frameworks/Python.framework/Versions/Current/bin:/usr/bin:/bin:/usr/sbin:/sbin" + steps: + + - name: CPU info + run: sysctl -a | grep brand + + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + allow-prereleases: true + python-version: "${{ matrix.python.version }}" + + - uses: dtolnay/rust-toolchain@master + with: + toolchain: "${{ env.RUST_TOOLCHAIN }}" + targets: "aarch64-apple-darwin" + components: "rust-src" + + - name: Build environment + run: | + cargo fetch --target aarch64-apple-darwin & + + export PATH=$HOME/.cargo/bin:$HOME/.local/bin:$PATH + + curl -LsSf https://astral.sh/uv/install.sh | sh + uv venv --python python${{ matrix.python.version }} + uv pip install --upgrade "maturin>=1.13.1,<2" -r test/requirements.txt -r integration/requirements.txt + + mkdir .cargo + cp ci/config.toml .cargo/config.toml + + - name: maturin + run: | + export PATH=$HOME/.cargo/bin:$HOME/.local/bin:$PATH + + MACOSX_DEPLOYMENT_TARGET="${{ matrix.python.macosx_target }}" \ + PYO3_CROSS_LIB_DIR=$(python -c "import sysconfig;print(sysconfig.get_config_var('LIBDIR'))") \ + maturin build \ + --release \ + --strip \ + --features=generic_simd,no_panic,optimize \ + --interpreter python${{ matrix.python.version }} \ + --target=aarch64-apple-darwin + uv pip install target/wheels/orjson*.whl + + - run: pytest -v test + env: + PYTHONMALLOC: "debug" + + - run: source .venv/bin/activate && ./integration/run thread + - run: source .venv/bin/activate && ./integration/run http + - run: source .venv/bin/activate && ./integration/run init + + - name: Store wheels + if: matrix.python.publish == true + uses: actions/upload-artifact@v7 + with: + name: orjson_macos_aarch64_${{ matrix.python.version }} + path: target/wheels + overwrite: true + retention-days: 1 + if-no-files-found: "error" + compression-level: 0 + + macos_universal2_aarch64: + name: "macos_universal2_python${{ matrix.python.version }}" + runs-on: macos-26 + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python: [ + { version: '3.14', macosx_target: "10.15", publish: true }, + { version: '3.13', macosx_target: "10.15", publish: true }, + { version: '3.12', macosx_target: "10.15", publish: true }, + { version: '3.11', macosx_target: "10.15", publish: true }, + { version: '3.10', macosx_target: "10.15", publish: true }, + ] + env: + CC: "clang" + CFLAGS: "-O2 -fstrict-aliasing" + LDFLAGS: "-Wl,--as-needed" + CFLAGS_x86_64_apple_darwin: "-O2 -fstrict-aliasing -fno-plt -march=x86-64-v2 -mtune=generic" + CFLAGS_aarch64_apple_darwin: "-O2 -fstrict-aliasing -fno-plt -mcpu=apple-m1 -mtune=generic" + RUSTFLAGS: "-Z unstable-options -C panic=immediate-abort -Z mir-opt-level=4 -Z threads=3 -D warnings" + PATH: "/Users/runner/work/orjson/orjson/.venv/bin:/Users/runner/.cargo/bin:/usr/local/opt/curl/bin:/usr/local/bin:/usr/local/sbin:/Users/runner/bin:/Library/Frameworks/Python.framework/Versions/Current/bin:/usr/bin:/bin:/usr/sbin:/sbin" + steps: + + - name: CPU info + run: sysctl -a | grep brand + + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + allow-prereleases: true + python-version: "${{ matrix.python.version }}" + + - uses: dtolnay/rust-toolchain@master + with: + toolchain: "${{ env.RUST_TOOLCHAIN }}" + targets: "aarch64-apple-darwin, x86_64-apple-darwin" + components: "rust-src" + + - name: Build environment + run: | + cargo fetch --target aarch64-apple-darwin & + + export PATH=$HOME/.cargo/bin:$HOME/.local/bin:$PATH + + curl -LsSf https://astral.sh/uv/install.sh | sh + uv venv --python python${{ matrix.python.version }} + uv pip install --upgrade "maturin>=1.13.1,<2" -r test/requirements.txt -r integration/requirements.txt + + mkdir .cargo + cp ci/config.toml .cargo/config.toml + + - name: maturin + run: | + export PATH=$HOME/.cargo/bin:$HOME/.local/bin:$PATH + + MACOSX_DEPLOYMENT_TARGET="${{ matrix.python.macosx_target }}" \ + PYO3_CROSS_LIB_DIR=$(python -c "import sysconfig;print(sysconfig.get_config_var('LIBDIR'))") \ + maturin build \ + --release \ + --strip \ + --features=generic_simd,no_panic,optimize \ + --interpreter python${{ matrix.python.version }} \ + --target=universal2-apple-darwin + uv pip install target/wheels/orjson*.whl + + - run: pytest -v test + env: + PYTHONMALLOC: "debug" + + - run: source .venv/bin/activate && ./integration/run thread + - run: source .venv/bin/activate && ./integration/run http + - run: source .venv/bin/activate && ./integration/run init + + - name: Store wheels + if: matrix.python.publish == true + uses: actions/upload-artifact@v7 + with: + name: orjson_universal2_aarch64_${{ matrix.python.version }} + path: target/wheels + overwrite: true + retention-days: 1 + if-no-files-found: "error" + compression-level: 0 + + windows_amd64: + name: "windows_amd64_python${{ matrix.python.version }}" + runs-on: windows-2025 + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python: [ + { version: '3.14', publish: true }, + { version: '3.13', publish: true }, + { version: '3.12', publish: true }, + { version: '3.11', publish: true }, + { version: '3.10', publish: true }, + ] + platform: [ + { arch: "x64", target: "x86_64-pc-windows-msvc", features: "avx512,no_panic,optimize" }, + { arch: "x86", target: "i686-pc-windows-msvc", features: "no_panic,optimize" }, + ] + env: + CFLAGS: "-O2" + LDFLAGS: "-Wl,--as-needed" + RUSTFLAGS: "-Z unstable-options -C panic=immediate-abort -Z mir-opt-level=4 -D warnings" + steps: + + - name: CPU info + shell: pwsh + run: Get-WmiObject -Class Win32_Processor -ComputerName. | Select-Object -Property Name, NumberOfCores, NumberOfLogicalProcessors + + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + allow-prereleases: true + python-version: "${{ matrix.python.version }}" + architecture: "${{ matrix.platform.arch }}" + + - uses: dtolnay/rust-toolchain@master + with: + toolchain: "${{ env.RUST_TOOLCHAIN }}" + targets: "${{ matrix.platform.target }}" + components: "rust-src" + + - name: Build environment + run: | + cargo fetch --target "${{ matrix.platform.target }}" & + + python.exe -m pip install --upgrade pip "maturin>=1.13.1,<2" wheel + python.exe -m pip install -r test\requirements.txt + + mkdir .cargo + cp ci\config.toml .cargo\config.toml + + - name: maturin + run: | + maturin.exe build --release --strip --features="${{ matrix.platform.features }}" --target="${{ matrix.platform.target }}" + python.exe -m pip install orjson --no-index --find-links target\wheels + + - run: python.exe -m pytest -s -rxX -v test + env: + PYTHONMALLOC: "debug" + + - name: Store wheels + if: matrix.python.publish == true + uses: actions/upload-artifact@v7 + with: + name: orjson_windows_amd64_${{ matrix.platform.arch }}_${{ matrix.python.version }} + path: target\wheels + overwrite: true + retention-days: 1 + if-no-files-found: "error" + compression-level: 0 + + windows_aarch64: + name: "windows_aarch64_python${{ matrix.python.version }}" + runs-on: windows-11-arm + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python: [ + { version: '3.14', publish: true }, + { version: '3.13', publish: true }, + { version: '3.12', publish: true }, + { version: '3.11', publish: true }, + ] + env: + CFLAGS: "-O2" + LDFLAGS: "-Wl,--as-needed" + RUSTFLAGS: "-Z unstable-options -C panic=immediate-abort -Z mir-opt-level=4 -D warnings" + TARGET: "aarch64-pc-windows-msvc" + steps: + + - name: CPU info + shell: pwsh + run: Get-WmiObject -Class Win32_Processor -ComputerName. | Select-Object -Property Name, NumberOfCores, NumberOfLogicalProcessors + + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + allow-prereleases: true + python-version: "${{ matrix.python.version }}" + architecture: "arm64" + + # from maturin + - shell: pwsh + run: | + Invoke-WebRequest -Uri "https://static.rust-lang.org/rustup/dist/$env:TARGET/rustup-init.exe" -OutFile rustup-init.exe + .\rustup-init.exe --default-toolchain "$env:RUST_TOOLCHAIN-$env:TARGET" --profile minimal --component rust-src -y + "$env:USERPROFILE\.cargo\bin" | Out-File -Append -Encoding ascii $env:GITHUB_PATH + "CARGO_HOME=$env:USERPROFILE\.cargo" | Out-File -Append -Encoding ascii $env:GITHUB_ENV + + - name: Build environment + run: | + cargo fetch --target "$" & + + python.exe -m sysconfig + python.exe -m pip install --upgrade pip "maturin>=1.13.1,<2" wheel + python.exe -m pip install -r test\requirements.txt + + mkdir .cargo + cp ci\config.toml .cargo\config.toml + + - name: maturin + run: | + maturin.exe build --release --strip --features=generic_simd,no_panic,optimize --target="$env:TARGET" + python.exe -m pip install orjson --no-index --find-links target\wheels + + - run: python.exe -m pytest -s -rxX -v test + env: + PYTHONMALLOC: "debug" + + - name: Store wheels + if: matrix.python.publish == true + uses: actions/upload-artifact@v7 + with: + name: orjson_windows_aarch64_${{ matrix.python.version }} + path: target\wheels + overwrite: true + retention-days: 1 + if-no-files-found: "error" + compression-level: 0 + + pypi: + name: PyPI + runs-on: ubuntu-24.04 + timeout-minutes: 10 + needs: [ + macos_aarch64, + macos_universal2_aarch64, + manylinux, + manylinux_cross, + musllinux_aarch64, + musllinux_amd64, + sdist, + windows_aarch64, + windows_amd64, + ] + environment: + name: PyPI + url: https://pypi.org/p/orjson + permissions: + id-token: write + steps: + - uses: actions/checkout@v6 + + - uses: actions/download-artifact@v8 + with: + merge-multiple: true + path: dist/ + pattern: orjson_* + + - run: ls -1 dist/ + + - uses: actions/setup-python@v6 + with: + python-version: "3.14" + + - run: ./script/check-pypi dist + + - name: Publish distribution to PyPI + if: "startsWith(github.ref, 'refs/tags/')" + uses: pypa/gh-action-pypi-publish@release/v1 + with: + attestations: true + packages-dir: dist + skip-existing: true + verbose: true diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 00000000..83b777d8 --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2023-2025) + +name: lint +on: push +env: + FORCE_COLOR: "1" + PIP_DISABLE_PIP_VERSION_CHECK: "1" +jobs: + lint: + runs-on: ubuntu-24.04 + steps: + - uses: actions/setup-python@v6 + with: + python-version: "3.14" + + - uses: actions/checkout@v6 + + - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain=stable --profile=default -y + - run: pip install -r requirements-lint.txt + + - run: cargo fmt + - run: ./script/lint + + - run: git diff --exit-code diff --git a/.github/workflows/linux-cross.yaml b/.github/workflows/linux-cross.yaml deleted file mode 100644 index 94e5569f..00000000 --- a/.github/workflows/linux-cross.yaml +++ /dev/null @@ -1,75 +0,0 @@ -name: linux-cross -on: - push: - branches: - - '*' - tags: - - '*' -jobs: - linux-cross: - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - python: [ - { version: '3.7', abi: 'cp37-cp37m' }, - { version: '3.8', abi: 'cp38-cp38' }, - { version: '3.9', abi: 'cp39-cp39' }, - { version: '3.10', abi: 'cp310-cp310' }, - ] - target: [armv7] - steps: - - uses: actions/checkout@v2 - - name: Build Wheels - uses: messense/maturin-action@v1 - env: - PYO3_CROSS_LIB_DIR: /opt/python/${{ matrix.python.abi }} - with: - maturin-version: v0.12.19 - target: ${{ matrix.target }} - rust-toolchain: nightly-2022-06-22 - manylinux: auto - args: -i python3.9 --release --strip --out dist --no-sdist - - uses: uraimo/run-on-arch-action@v2.0.5 - name: Install built wheel - with: - arch: ${{ matrix.target }} - distro: ubuntu20.04 - githubToken: ${{ github.token }} - install: | - apt-get update - apt-get install -y --no-install-recommends python3-dev python3-venv software-properties-common build-essential - add-apt-repository ppa:deadsnakes/ppa - apt-get update - apt-get install -y curl python3.7-dev python3.7-venv python3.9-dev python3.9-venv python3.10-dev python3.10-venv - run: | - PYTHON=python${{ matrix.python.version }} - $PYTHON -m venv venv - venv/bin/pip install -U pip - venv/bin/pip install -r test/requirements.txt - venv/bin/pip install orjson --no-index --find-links dist/ --force-reinstall - venv/bin/python -m pytest -s -rxX -v test - - name: Upload wheels - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist - - release: - name: Release - runs-on: ubuntu-22.04 - if: "startsWith(github.ref, 'refs/tags/')" - needs: [ linux-cross ] - steps: - - uses: actions/download-artifact@v2 - with: - name: wheels - - uses: actions/setup-python@v2 - with: - python-version: "3.10" - - run: pip install "maturin>=0.12.19,<0.13" - - name: deploy - run: maturin upload --skip-existing --username "$MATURIN_USERNAME" *.whl - env: - MATURIN_USERNAME: ${{ secrets.TWINE_USERNAME }} - MATURIN_PASSWORD: ${{ secrets.TWINE_PASSWORD }} diff --git a/.github/workflows/manylinux2014.yaml b/.github/workflows/manylinux2014.yaml deleted file mode 100644 index d70a9ca2..00000000 --- a/.github/workflows/manylinux2014.yaml +++ /dev/null @@ -1,47 +0,0 @@ -name: manylinux2014 -on: - push: - branches: - - '*' - tags: - - '*' -jobs: - manylinux2014: - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - python: [ - { version: '3.7', abi: 'cp37-cp37m' }, - { version: '3.8', abi: 'cp38-cp38' }, - { version: '3.9', abi: 'cp39-cp39' }, - { version: '3.10', abi: 'cp310-cp310' }, - ] - env: - PATH: /github/home/.local/bin:/github/home/.cargo/bin:/opt/python/${{ matrix.python.abi }}/bin:/opt/rh/devtoolset-10/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - CC: "gcc" - CFLAGS: "-O2 -fno-plt -flto" - LDFLAGS: "-O2 -flto -Wl,--as-needed" - container: - image: quay.io/pypa/manylinux2014_x86_64:latest - options: --user 0 - steps: - - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2022-06-22 --profile minimal -y - - uses: actions/checkout@v2 - - run: python3 -m pip install --user --upgrade pip "maturin>=0.12.19,<0.13" wheel - - run: cargo fetch - - run: maturin build --no-sdist --release --strip --cargo-extra-args="--features=unstable-simd,yyjson" --compatibility manylinux2014 --interpreter python${{ matrix.python.version }} - - run: python3 -m pip install --user target/wheels/orjson*.whl - - run: python3 -m pip install --user -r test/requirements.txt -r integration/requirements.txt - - run: pytest -s -rxX -v test - - run: python3 -m pip uninstall -y numpy - - run: pytest -s -rxX -v test - - run: ./integration/run thread - - run: ./integration/run http - - run: git config --global --add safe.directory /__w/orjson/orjson - - name: deploy - run: maturin upload --skip-existing --username "$MATURIN_USERNAME" target/wheels/orjson-*.whl - if: "startsWith(github.ref, 'refs/tags/')" - env: - MATURIN_USERNAME: ${{ secrets.TWINE_USERNAME }} - MATURIN_PASSWORD: ${{ secrets.TWINE_PASSWORD }} diff --git a/.github/workflows/manylinux_2_28.yaml b/.github/workflows/manylinux_2_28.yaml deleted file mode 100644 index a0933b96..00000000 --- a/.github/workflows/manylinux_2_28.yaml +++ /dev/null @@ -1,51 +0,0 @@ -name: manylinux_2_28 -on: - push: - branches: - - '*' - tags: - - '*' -jobs: - manylinux_2_28: - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - python: [ - { version: '3.7', abi: 'cp37-cp37m' }, - { version: '3.8', abi: 'cp38-cp38' }, - { version: '3.9', abi: 'cp39-cp39' }, - { version: '3.10', abi: 'cp310-cp310' }, - { version: '3.11', abi: 'cp311-cp311' }, - ] - env: - PATH: /github/home/.local/bin:/github/home/.cargo/bin:/opt/python/${{ matrix.python.abi }}/bin:/opt/rh/gcc-toolset-11/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - container: - image: quay.io/pypa/manylinux_2_28_x86_64:latest - options: --user 0 - steps: - - run: yum update -y && yum install -y clang lld - - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2022-06-22 --profile minimal -y - - uses: actions/checkout@v2 - - run: python3 -m pip install --user --upgrade pip "maturin>=0.12.19,<0.13" wheel - - run: cargo fetch - - run: maturin build --no-sdist --release --strip --cargo-extra-args="--features=unstable-simd,yyjson" --compatibility manylinux_2_28 --interpreter python${{ matrix.python.version }} - env: - CC: "clang" - CFLAGS: "-O2 -fno-plt -flto=thin" - LDFLAGS: "-O2 -flto=thin -fuse-ld=lld -Wl,--as-needed" - RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=lld" - - run: python3 -m pip install --user target/wheels/orjson*.whl - - run: python3 -m pip install --user -r test/requirements.txt -r integration/requirements.txt - - run: pytest -s -rxX -v test - - run: python3 -m pip uninstall -y numpy - - run: pytest -s -rxX -v test - - run: ./integration/run thread - - run: ./integration/run http - - run: git config --global --add safe.directory /__w/orjson/orjson - - name: deploy - run: maturin upload --skip-existing --username "$MATURIN_USERNAME" target/wheels/orjson-*.whl - if: "startsWith(github.ref, 'refs/tags/')" - env: - MATURIN_USERNAME: ${{ secrets.TWINE_USERNAME }} - MATURIN_PASSWORD: ${{ secrets.TWINE_PASSWORD }} diff --git a/.github/workflows/musllinux.yaml b/.github/workflows/musllinux.yaml deleted file mode 100644 index 0783a96f..00000000 --- a/.github/workflows/musllinux.yaml +++ /dev/null @@ -1,86 +0,0 @@ -name: musllinux -on: - push: - branches: - - '*' - tags: - - '*' -jobs: - musllinux: - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - python: [ - { version: '3.7' }, - { version: '3.8' }, - { version: '3.9' }, - { version: '3.10' }, - ] - platform: - - target: aarch64-unknown-linux-musl - arch: aarch64 - - target: x86_64-unknown-linux-musl - arch: x86_64 - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python.version }} - - name: Build wheels - uses: messense/maturin-action@v1 - with: - maturin-version: v0.12.19 - rust-toolchain: nightly-2022-06-22 - target: ${{ matrix.platform.target }} - manylinux: musllinux_1_1 - args: --release --strip --out dist --no-sdist --cargo-extra-args="--features=unstable-simd,yyjson" -i python${{ matrix.python.version }} - - name: Set up QEMU - if: matrix.platform.arch != 'x86_64' - uses: docker/setup-qemu-action@v2 - with: - image: tonistiigi/binfmt:qemu-v6.2.0 - platforms: all - - name: Install built wheel - uses: addnab/docker-run-action@v3 - with: - image: quay.io/pypa/musllinux_1_1_${{ matrix.platform.arch }}:latest - options: -v ${{ github.workspace }}:/io -w /io - run: | - # workaround zoneinfo._common.ZoneInfoNotFoundError: 'No time zone found with key UTC' - # exception when running tests - apk add tzdata - - # Don't install numpy since there are no musllinux wheels - sed -i '/^numpy/d' test/requirements.txt - - PYTHON=python${{ matrix.python.version }} - $PYTHON -m venv venv - venv/bin/pip install -U pip - venv/bin/pip install -r test/requirements.txt - venv/bin/pip install orjson --no-index --find-links dist/ --force-reinstall - venv/bin/python -m pytest -s -rxX -v test - - name: Upload wheels - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist - - release: - name: Release - runs-on: ubuntu-22.04 - if: "startsWith(github.ref, 'refs/tags/')" - needs: [ musllinux ] - steps: - - uses: actions/download-artifact@v2 - with: - name: wheels - - uses: actions/setup-python@v2 - with: - python-version: "3.10" - - run: pip install "maturin>=0.12.19,<0.13" - - name: deploy - run: maturin upload --skip-existing --username "$MATURIN_USERNAME" *.whl - env: - MATURIN_USERNAME: ${{ secrets.TWINE_USERNAME }} - MATURIN_PASSWORD: ${{ secrets.TWINE_PASSWORD }} diff --git a/.github/workflows/unusual.yaml b/.github/workflows/unusual.yaml new file mode 100644 index 00000000..5550e498 --- /dev/null +++ b/.github/workflows/unusual.yaml @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2023-2026) + +name: unusual +on: push +env: + FORCE_COLOR: "1" + PIP_DISABLE_PIP_VERSION_CHECK: "1" +jobs: + + unusual: + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + cfg: [ + { rust: "1.95", python: "3.15", version_check: "1" }, + { rust: "1.95", python: "3.14", version_check: "0" }, + { rust: "1.95", python: "3.10", version_check: "0" }, + ] + steps: + - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain ${{ matrix.cfg.rust }} --profile minimal -y + + - uses: actions/setup-python@v5 + with: + python-version: '${{ matrix.cfg.python }}' + allow-prereleases: true + + - run: python -m pip install --user --upgrade pip "maturin>=1.13.1,<2" wheel + + - uses: actions/checkout@v6 + + - name: build + run: | + PATH="$HOME/.cargo/bin:$PATH" UNSAFE_PYO3_SKIP_VERSION_CHECK="${{ matrix.cfg.version_check }}" \ + maturin build \ + --profile=dev \ + --interpreter python${{ matrix.cfg.python }} \ + --target=x86_64-unknown-linux-gnu + + - run: python -m pip install --user target/wheels/orjson*.whl + - run: python -m pip install --user -r test/requirements.txt -r integration/requirements.txt + + - run: pytest -s -rxX -v test + timeout-minutes: 4 + env: + PYTHONMALLOC: "debug" + + - run: ./integration/run thread + timeout-minutes: 2 + + - run: ./integration/run http + timeout-minutes: 2 + + - run: ./integration/run init + timeout-minutes: 2 diff --git a/.gitignore b/.gitignore index e032ba0f..fdf3c2de 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,12 @@ *.patch +/.benchmarks +/.coverage +/.mypy_cache +/.pytest_cache +/.venv* +/build +/include/cargo /perf.* -__pycache__ -*.pyc /target -.pytest_cache -/.venv* -benchmark_*.svg -.coverage -.benchmarks -.mypy_cache -vendor -build -!json/perfect_float.patch +/vendor +__pycache__ diff --git a/CHANGELOG.md b/CHANGELOG.md index 4576773c..593fb47b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,605 @@ # Changelog + +## 3.11.9 - 2026-05-06 + +### Changed + +- Build now depends on Rust 1.95 or later instead of 1.89. + +### Fixed + +- Fix building on Rust 1.95. + + +## 3.11.8 - 2026-03-31 + +### Changed + +- Build and compatibility improvements. + + +## 3.11.7 - 2026-02-02 + +### Changed + +- Use a faster library to serialize `float`. Users with byte-exact regression +tests should note positive exponents are now written using a `+`, e.g., +`1.2e+30` instead of `1.2e30`. Both formats are spec-compliant. +- ABI compatibility with CPython 3.15 alpha 5 free-threading. + + +## 3.11.6 - 2026-01-29 + +### Changed + +- orjson now includes code licensed under the Mozilla Public License 2.0 (MPL-2.0). +- Drop support for Python 3.9. +- ABI compatibility with CPython 3.15 alpha 5. +- Build now depends on Rust 1.89 or later instead of 1.85. + +### Fixed + +- Fix sporadic crash serializing deeply nested `list` of `dict`. + + +## 3.11.5 - 2025-12-06 + +### Changed + +- Show simple error message instead of traceback when attempting to +build on unsupported Python versions. + + +## 3.11.4 - 2025-10-24 + +### Changed + +- ABI compatibility with CPython 3.15 alpha 1. +- Publish PyPI wheels for 3.14 and manylinux i686, manylinux arm7, +manylinux ppc64le, manylinux s390x. +- Build now requires a C compiler. + + +## 3.11.3 - 2025-08-26 + +### Fixed + +- Fix PyPI project metadata when using maturin 1.9.2 or later. + + +## 3.11.2 - 2025-08-12 + +### Fixed + +- Fix build using Rust 1.89 on amd64. + +### Changed + +- Build now depends on Rust 1.85 or later instead of 1.82. + + +## 3.11.1 - 2025-07-25 + +### Changed + +- Publish PyPI wheels for CPython 3.14. + +### Fixed + +- Fix `str` on big-endian architectures. This was introduced in 3.11.0. + + +## 3.11.0 - 2025-07-15 + +### Changed + +- Use a deserialization buffer allocated per request instead of a shared +buffer allocated on import. +- ABI compatibility with CPython 3.14 beta 4. + + +## 3.10.18 - 2025-04-29 + +### Fixed + +- Fix incorrect escaping of the vertical tabulation character. This was +introduced in 3.10.17. + + +## 3.10.17 - 2025-04-29 + +### Changed + +- Publish PyPI Windows aarch64/arm64 wheels. +- ABI compatibility with CPython 3.14 alpha 7. +- Fix incompatibility running on Python 3.13 using WASM. + + +## 3.10.16 - 2025-03-24 + +### Changed + +- Improve performance of serialization on amd64 machines with AVX-512. +- ABI compatibility with CPython 3.14 alpha 6. +- Drop support for Python 3.8. +- Publish additional PyPI wheels for macOS that target only aarch64, macOS 15, +and recent Python. + + +## 3.10.15 - 2025-01-08 + +### Changed + +- Publish PyPI manylinux aarch64 wheels built and tested on aarch64. +- Publish PyPI musllinux aarch64 and arm7l wheels built and tested on aarch64. +- Publish PyPI manylinux Python 3.13 wheels for i686, arm7l, ppc64le, and s390x. + + +## 3.10.14 - 2024-12-29 + +### Changed + +- Specify build system dependency on `maturin>=1,<2` again. +- Allocate memory using `PyMem_Malloc()` and similar APIs for integration +with pymalloc, mimalloc, and tracemalloc. +- Source distribution does not ship compressed test documents and relevant +tests skip if fixtures are not present. +- Build now depends on Rust 1.82 or later instead of 1.72. + + +## 3.10.13 - 2024-12-29 + +### Changed + +- Fix compatibility with maturin introducing a breaking change in 1.8.0 and +specify a fixed version of maturin. Projects relying on any previous version +being buildable from source by end users (via PEP 517) must upgrade to at +least this version. + + +## 3.10.12 - 2024-11-23 + +### Changed + +- Publish PyPI manylinux i686 wheels. +- Publish PyPI musllinux i686 and arm7l wheels. +- Publish PyPI macOS wheels for Python 3.10 or later built on macOS 15. +- Publish PyPI Windows wheels using trusted publishing. + + +## 3.10.11 - 2024-11-01 + +### Changed + +- Improve performance of UUIDs. +- Publish PyPI wheels with trusted publishing and PEP 740 attestations. +- Include text of licenses for vendored dependencies. + + +## 3.10.10 - 2024-10-22 + +### Fixed + +- Fix `int` serialization on `s390x`. This was introduced in 3.10.8. + +### Changed + +- Publish aarch64 manylinux_2_17 wheel for 3.13 to PyPI. + + +## 3.10.9 - 2024-10-19 + +### Fixed + +- Fix `int` serialization on 32-bit Python 3.8, 3.9, 3.10. This was +introduced in 3.10.8. + + +## 3.10.8 - 2024-10-19 + +### Changed + +- `int` serialization no longer chains `OverflowError` to the +the `__cause__` attribute of `orjson.JSONEncodeError` when range exceeded. +- Compatibility with CPython 3.14 alpha 1. +- Improve performance. + + +## 3.10.7 - 2024-08-08 + +### Changed + +- Improve performance of stable Rust amd64 builds. + + +## 3.10.6 - 2024-07-02 + +### Changed + +- Improve performance. + + +## 3.10.5 - 2024-06-13 + +### Changed + +- Improve performance. + + +## 3.10.4 - 2024-06-10 + +### Changed + +- Improve performance. + + +## 3.10.3 - 2024-05-03 + +### Changed + +- `manylinux` amd64 builds include runtime-detected AVX-512 `str` +implementation. +- Tests now compatible with numpy v2. + + +## 3.10.2 - 2024-05-01 + +### Fixed + +- Fix crash serializing `str` introduced in 3.10.1. + +### Changed + +- Improve performance. +- Drop support for arm7. + + +## 3.10.1 - 2024-04-15 + +### Fixed + +- Serializing `numpy.ndarray` with non-native endianness raises +`orjson.JSONEncodeError`. + +### Changed + +- Improve performance of serializing. + + +## 3.10.0 - 2024-03-27 + +### Changed + +- Support serializing `numpy.float16` (`numpy.half`). +- sdist uses metadata 2.3 instead of 2.1. +- Improve Windows PyPI builds. + + +## 3.9.15 - 2024-02-23 + +### Fixed + +- Implement recursion limit of 1024 on `orjson.loads()`. +- Use byte-exact read on `str` formatting SIMD path to avoid crash. + + +## 3.9.14 - 2024-02-14 + +### Fixed + +- Fix crash serializing `str` introduced in 3.9.11. + +### Changed + +- Build now depends on Rust 1.72 or later. + + +## 3.9.13 - 2024-02-03 + +### Fixed + +- Serialization `str` escape uses only 128-bit SIMD. +- Fix compatibility with CPython 3.13 alpha 3. + +### Changed + +- Publish `musllinux_1_2` instead of `musllinux_1_1` wheels. +- Serialization uses small integer optimization in CPython 3.12 or later. + + +## 3.9.12 - 2024-01-18 + +### Changed + +- Update benchmarks in README. + +### Fixed + +- Minimal `musllinux_1_1` build due to sporadic CI failure. + + +## 3.9.11 - 2024-01-18 + +### Changed + +- Improve performance of serializing. `str` is significantly faster. Documents +using `dict`, `list`, and `tuple` are somewhat faster. + + +## 3.9.10 - 2023-10-26 + +### Fixed + +- Fix debug assert failure on 3.12 `--profile=dev` build. + + +## 3.9.9 - 2023-10-12 + +### Changed + +- `orjson` module metadata explicitly marks subinterpreters as not supported. + + +## 3.9.8 - 2023-10-10 + +### Changed + +- Improve performance. +- Drop support for Python 3.7. + + +## 3.9.7 - 2023-09-08 + +### Fixed + +- Fix crash in `orjson.loads()` due to non-reentrant handling of persistent +buffer. This was introduced in 3.9.3. +- Handle some FFI removals in CPython 3.13. + + +## 3.9.6 - 2023-09-07 + +### Fixed + +- Fix numpy reference leak on unsupported array dtype. +- Fix numpy.datetime64 reference handling. + +### Changed + +- Minor performance improvements. + + +## 3.9.5 - 2023-08-16 + +### Fixed + +- Remove futex from module import and initialization path. + + +## 3.9.4 - 2023-08-07 + +### Fixed + +- Fix hash builder using default values. +- Fix non-release builds of orjson copying large deserialization buffer +from stack to heap. This was introduced in 3.9.3. + + +## 3.9.3 - 2023-08-06 + +### Fixed + +- Fix compatibility with CPython 3.12. + +### Changed + +- Support i686/x86 32-bit Python installs on Windows. + + +## 3.9.2 - 2023-07-07 + +### Fixed + +- Fix the `__cause__` exception on `orjson.JSONEncodeError` possibly being +denormalized, i.e., of type `str` instead of `Exception`. + + +## 3.9.1 - 2023-06-09 + +### Fixed + +- Fix memory leak on chained tracebacks of exceptions raised in `default`. This +was introduced in 3.8.12. + + +## 3.9.0 - 2023-06-01 + +### Added + +- `orjson.Fragment` includes already-serialized JSON in a document. + + +## 3.8.14 - 2023-05-25 + +### Changed + +- PyPI `manylinux` wheels are compiled for `x86-64` instead of `x86-64-v2`. + + +## 3.8.13 - 2023-05-23 + +### Changed + +- Source distribution contains all source code required for an offline build. +- PyPI macOS wheels use a `MACOSX_DEPLOYMENT_TARGET` of 10.15 instead of 11. +- Build uses maturin v1. + + +## 3.8.12 - 2023-05-07 + +### Changed + +- Exceptions raised in `default` are now chained as the `__cause__` attribute +on `orjson.JSONEncodeError`. + + +## 3.8.11 - 2023-04-27 + +### Changed + +- `orjson.loads()` on an empty document has a specific error message. +- PyPI `manylinux_2_28_x86_64` wheels are compiled for `x86-64-v2`. +- PyPI macOS wheels are only `universal2` and compiled for +`x86-64-v2` and `apple-m1`. + + +## 3.8.10 - 2023-04-09 + +### Fixed + +- Fix compatibility with CPython 3.12.0a7. +- Fix compatibility with big-endian architectures. +- Fix crash in serialization. + +### Changed + +- Publish musllinux 3.11 wheels. +- Publish s390x wheels. + + +## 3.8.9 - 2023-03-28 + +### Fixed + +- Fix parallel initialization of orjson. + + +## 3.8.8 - 2023-03-20 + +### Changed + +- Publish ppc64le wheels. + + +## 3.8.7 - 2023-02-28 + +### Fixed + +- Use serialization backend introduced in 3.8.4 only on well-tested +platforms such as glibc, macOS by default. + + +## 3.8.6 - 2023-02-09 + +### Fixed + +- Fix crash serializing when using musl libc. + +### Changed + +- Make `python-dateutil` optional in tests. +- Handle failure to load system timezones in tests. + + +## 3.8.5 - 2023-01-10 + +### Fixed + +- Fix `orjson.dumps()` invalid output on Windows. + + +## 3.8.4 - 2023-01-04 + +### Changed + +- Improve performance. + + +## 3.8.3 - 2022-12-02 + +### Fixed + +- `orjson.dumps()` accepts `option=None` per `Optional[int]` type. + + +## 3.8.2 - 2022-11-20 + +### Fixed + +- Fix tests on 32-bit for `numpy.intp` and `numpy.uintp`. + +### Changed + +- Build now depends on rustc 1.60 or later. +- Support building with maturin 0.13 or 0.14. + + +## 3.8.1 - 2022-10-25 + +### Changed + +- Build maintenance for Python 3.11. + + +## 3.8.0 - 2022-08-27 + +### Changed + +- Support serializing `numpy.int16` and `numpy.uint16`. + + +## 3.7.12 - 2022-08-14 + +### Fixed + +- Fix datetime regression tests for tzinfo 2022b. + +### Changed + +- Improve performance. + + +## 3.7.11 - 2022-07-31 + +### Fixed + +- Revert `dict` iterator implementation introduced in 3.7.9. + + +## 3.7.10 - 2022-07-30 + +### Fixed + +- Fix serializing `dict` with deleted final item. This was introduced in 3.7.9. + + +## 3.7.9 - 2022-07-29 + +### Changed + +- Improve performance of serializing. +- Improve performance of serializing pretty-printed (`orjson.OPT_INDENT_2`) +to be much nearer to compact. +- Improve performance of deserializing `str` input. +- orjson now requires Rust 1.57 instead of 1.54 to build. + + +## 3.7.8 - 2022-07-19 + +### Changed + +- Build makes best effort instead of requiring "--features". +- Build using maturin 0.13. + + ## 3.7.7 - 2022-07-06 ### Changed - Support Python 3.11. + ## 3.7.6 - 2022-07-03 ### Changed @@ -13,6 +607,7 @@ - Handle unicode changes in CPython 3.12. - Build PyPI macOS wheels on 10.15 instead of 12 for compatibility. + ## 3.7.5 - 2022-06-28 ### Fixed @@ -20,6 +615,7 @@ - Fix issue serializing dicts that had keys popped and replaced. This was introduced in 3.7.4. + ## 3.7.4 - 2022-06-28 ### Changed @@ -30,19 +626,24 @@ introduced in 3.7.4. - Fix deallocation of `orjson.JSONDecodeError`. + ## 3.7.3 - 2022-06-23 + ## Changed - Improve build. - Publish aarch64 musllinux wheels. + ## 3.7.2 - 2022-06-07 + ## Changed - Improve deserialization performance. + ## 3.7.1 - 2022-06-03 ### Fixed @@ -51,6 +652,7 @@ introduced in 3.7.4. `json.JSONDecodeError` instead of `ValueError` - Null-terminate the internal buffer of `orjson.dumps()` output. + ## 3.7.0 - 2022-06-03 ### Changed @@ -59,12 +661,14 @@ introduced in 3.7.4. backend. PyPI wheels for manylinux_2_28 and macOS have it enabled. Packagers are advised to see the README. + ## 3.6.9 - 2022-06-01 ### Changed - Improve serialization and deserialization performance. + ## 3.6.8 - 2022-04-15 ### Fixed @@ -72,6 +676,7 @@ are advised to see the README. - Fix serialization of `numpy.datetime64("NaT")` to raise on an unsupported type. + ## 3.6.7 - 2022-02-14 ### Changed @@ -84,6 +689,7 @@ unsupported type. - Fix build requiring `python` on `PATH`. + ## 3.6.6 - 2022-01-21 ### Changed @@ -98,6 +704,7 @@ are `zoneinfo.ZoneInfo`. - Fix `orjson.OPT_STRICT_INTEGER` not raising an error on values exceeding a 64-bit integer maximum. + ## 3.6.5 - 2021-12-05 ### Fixed @@ -105,6 +712,7 @@ values exceeding a 64-bit integer maximum. - Fix build on macOS aarch64 CPython 3.10. - Fix build issue on 32-bit. + ## 3.6.4 - 2021-10-01 ### Fixed @@ -114,12 +722,14 @@ using `__slots__`. - Decrement refcount for numpy `PyArrayInterface`. - Fix build on recent versions of Rust nightly. + ## 3.6.3 - 2021-08-20 ### Fixed - Fix build on aarch64 using the Rust stable channel. + ## 3.6.2 - 2021-08-17 ### Changed @@ -132,6 +742,7 @@ usage is now disabled by default and packagers are advised to add implementations that use AVX2 or SSE4.2. - Drop support for Python 3.6. + ## 3.6.1 - 2021-08-04 ### Changed @@ -143,6 +754,7 @@ implementations that use AVX2 or SSE4.2. - Fix compilation on latest Rust nightly. + ## 3.6.0 - 2021-07-08 ### Added @@ -150,6 +762,7 @@ implementations that use AVX2 or SSE4.2. - `orjson.dumps()` serializes `numpy.datetime64` instances as RFC 3339 strings. + ## 3.5.4 - 2021-06-30 ### Fixed @@ -162,6 +775,7 @@ without default specified. - Publish python3.10 and python3.9 manylinux_2_24 wheels. + ## 3.5.3 - 2021-06-01 ### Fixed @@ -169,6 +783,7 @@ without default specified. - `orjson.JSONDecodeError` now has `pos`, `lineno`, and `colno`. - Fix build on recent versions of Rust nightly. + ## 3.5.2 - 2021-04-15 ### Changed @@ -176,12 +791,14 @@ without default specified. - Improve serialization and deserialization performance. - `orjson.dumps()` serializes individual `numpy.bool_` objects. + ## 3.5.1 - 2021-03-06 ### Changed - Publish `universal2` wheels for macOS supporting Apple Silicon (aarch64). + ## 3.5.0 - 2021-02-24 ### Added @@ -199,6 +816,7 @@ four digits. - `orjson.dumps()` when given a non-C contiguous `numpy.ndarray` has an error message suggesting to use `default`. + ## 3.4.8 - 2021-02-04 ### Fixed @@ -209,6 +827,7 @@ an error message suggesting to use `default`. - Fix build warnings on ppcle64. + ## 3.4.7 - 2021-01-19 ### Changed @@ -216,18 +835,21 @@ an error message suggesting to use `default`. - Use vectorcall APIs for method calls on python3.9 and above. - Publish python3.10 wheels for Linux on amd64 and aarch64. + ## 3.4.6 - 2020-12-07 ### Fixed - Fix compatibility with debug builds of CPython. + ## 3.4.5 - 2020-12-02 ### Fixed - Fix deserializing long strings on processors without AVX2. + ## 3.4.4 - 2020-11-25 ### Changed @@ -235,12 +857,14 @@ an error message suggesting to use `default`. - `orjson.dumps()` serializes integers up to a 64-bit unsigned integer's maximum. It was previously the maximum of a 64-bit signed integer. + ## 3.4.3 - 2020-10-30 ### Fixed - Fix regression in parsing similar `dict` keys. + ## 3.4.2 - 2020-10-29 ### Changed @@ -249,6 +873,7 @@ maximum. It was previously the maximum of a 64-bit signed integer. - Publish Windows python3.9 wheel. - Disable unsupported SIMD features on non-x86, non-ARM targets + ## 3.4.1 - 2020-10-20 ### Fixed @@ -261,6 +886,7 @@ maximum. It was previously the maximum of a 64-bit signed integer. - Publish macos python3.9 wheel. - More packaging documentation. + ## 3.4.0 - 2020-09-25 ### Added @@ -275,6 +901,7 @@ maximum. It was previously the maximum of a 64-bit signed integer. - No longer publish `manylinux1` wheels due to tooling dropping support. + ## 3.3.1 - 2020-08-17 ### Fixed @@ -287,6 +914,7 @@ was introduced in 3.2.0. - Publish `manylinux2014` wheels for amd64 in addition to `manylinux1`. + ## 3.3.0 - 2020-07-24 ### Added @@ -296,6 +924,7 @@ was introduced in 3.2.0. - `orjson.OPT_PASSTHROUGH_DATACLASS` causes `orjson.dumps()` to pass `dataclasses.dataclass` instances to `default`. + ## 3.2.2 - 2020-07-13 ### Fixed @@ -306,12 +935,14 @@ was introduced in 3.2.0. - Improve deserialization performance of `str`. + ## 3.2.1 - 2020-07-03 ### Fixed - Fix `orjson.dumps(..., **{})` raising `TypeError` on python3.6. + ## 3.2.0 - 2020-06-30 ### Added @@ -322,12 +953,14 @@ was introduced in 3.2.0. - Improve deserialization performance of `str`. + ## 3.1.2 - 2020-06-23 ### Fixed - Fix serializing zero-dimension `numpy.ndarray`. + ## 3.1.1 - 2020-06-20 ### Fixed @@ -335,6 +968,7 @@ was introduced in 3.2.0. - Fix repeated serialization of `str` that are ASCII-only and have a legacy (non-compact) layout. + ## 3.1.0 - 2020-06-08 ### Added @@ -346,6 +980,7 @@ output. `datetime` objects to `default` so the caller can customize the output. + ## 3.0.2 - 2020-05-27 ### Changed @@ -356,6 +991,7 @@ Python idiom that a leading underscores marks an attribute as "private." - `orjson.dumps()` does not serialize `dataclasses.dataclass` attributes that are `InitVar` or `ClassVar` whether using `__slots__` or not. + ## 3.0.1 - 2020-05-19 ### Fixed @@ -373,6 +1009,7 @@ garbage collector runs. calling convention on python3.7 and above. - Reduce build time. + ## 3.0.0 - 2020-05-01 ### Added @@ -385,12 +1022,14 @@ calling convention on python3.7 and above. instances by default. The options `OPT_SERIALIZE_DATACLASS` and `OPT_SERIALIZE_UUID` can still be specified but have no effect. + ## 2.6.8 - 2020-04-30 ### Changed - The source distribution vendors a forked dependency. + ## 2.6.7 - 2020-04-30 ### Fixed @@ -401,6 +1040,7 @@ instances by default. The options `OPT_SERIALIZE_DATACLASS` and - The source distribution sets the recommended RUSTFLAGS in `.cargo/config`. + ## 2.6.6 - 2020-04-24 ### Fixed @@ -409,6 +1049,7 @@ instances by default. The options `OPT_SERIALIZE_DATACLASS` and interpreter start time when not used. - Reduce build time by half. + ## 2.6.5 - 2020-04-08 ### Fixed @@ -416,12 +1057,14 @@ interpreter start time when not used. - Fix deserialization raising `JSONDecodeError` on some valid negative floats with large exponents. + ## 2.6.4 - 2020-04-08 ### Changed - Improve deserialization performance of floats by about 40%. + ## 2.6.3 - 2020-04-01 ### Changed @@ -429,6 +1072,7 @@ floats with large exponents. - Serialize `enum.Enum` objects. - Minor performance improvements. + ## 2.6.2 - 2020-03-27 ### Changed @@ -440,6 +1084,7 @@ floats with large exponents. - Fix compilation failure on 32-bit. + ## 2.6.1 - 2020-03-19 ### Changed @@ -447,6 +1092,7 @@ floats with large exponents. - Serialization is 10-20% faster and uses about 50% less memory by writing directly to the returned `bytes` object. + ## 2.6.0 - 2020-03-10 ### Added @@ -454,6 +1100,7 @@ directly to the returned `bytes` object. - `orjson.dumps()` pretty prints with an indentation of two spaces if `option=orjson.OPT_INDENT_2` is specified. + ## 2.5.2 - 2020-03-07 ### Changed @@ -461,6 +1108,7 @@ directly to the returned `bytes` object. - Publish `manylinux2014` wheels for `aarch64`. - numpy support now includes `numpy.uint32` and `numpy.uint64`. + ## 2.5.1 - 2020-02-24 ### Changed @@ -468,6 +1116,7 @@ directly to the returned `bytes` object. - `manylinux1` wheels for 3.6, 3.7, and 3.8 are now compliant with the spec by not depending on glibc 2.18. + ## 2.5.0 - 2020-02-19 ### Added @@ -475,6 +1124,7 @@ not depending on glibc 2.18. - `orjson.dumps()` serializes `dict` keys of type other than `str` if `option=orjson.OPT_NON_STR_KEYS` is specified. + ## 2.4.0 - 2020-02-14 ### Added @@ -487,6 +1137,7 @@ not depending on glibc 2.18. - Fix `dataclasses.dataclass` attributes that are `dict` to be effected by `orjson.OPT_SORT_KEYS`. + ## 2.3.0 - 2020-02-12 ### Added @@ -504,6 +1155,7 @@ specified. - Fix documentation on `default`, in particular documenting the need to raise an exception if the type cannot be handled. + ## 2.2.2 - 2020-02-10 ### Changed @@ -511,6 +1163,7 @@ an exception if the type cannot be handled. - Performance improvements to serializing a list containing elements of the same type. + ## 2.2.1 - 2020-02-04 ### Fixed @@ -522,6 +1175,7 @@ the decimal, e.g., `-2.`, `2.e-3`. - Build Linux, macOS, and Windows wheels on more recent distributions. + ## 2.2.0 - 2020-01-22 ### Added @@ -534,6 +1188,7 @@ the decimal, e.g., `-2.`, `2.e-3`. - Minor performance improvements. - Publish Python 3.9 wheel for Linux. + ## 2.1.4 - 2020-01-08 ### Fixed @@ -544,12 +1199,14 @@ the decimal, e.g., `-2.`, `2.e-3`. - Improve documentation. + ## 2.1.3 - 2019-11-12 ### Changed - Publish Python 3.8 wheels for macOS and Windows. + ## 2.1.2 - 2019-11-07 ### Changed @@ -557,12 +1214,14 @@ the decimal, e.g., `-2.`, `2.e-3`. - The recursion limit of `default` on `orjson.dumps()` has been increased from 5 to 254. + ## 2.1.1 - 2019-10-29 ### Changed - Publish `manylinux1` wheels instead of `manylinux2010`. + ## 2.1.0 - 2019-10-24 ### Added @@ -581,12 +1240,14 @@ instances. - Drop support for Python 3.5. - Publish `manylinux2010` wheels instead of `manylinux1`. + ## 2.0.11 - 2019-10-01 ### Changed - Publish Python 3.8 wheel for Linux. + ## 2.0.10 - 2019-09-25 ### Changed @@ -594,6 +1255,7 @@ instances. - Performance improvements and lower memory usage in deserialization by creating only one `str` object for repeated map keys. + ## 2.0.9 - 2019-09-22 ### Changed @@ -605,6 +1267,7 @@ by creating only one `str` object for repeated map keys. - Fix inaccurate zero padding in serialization of microseconds on `datetime.time` objects. + ## 2.0.8 - 2019-09-18 ### Fixed @@ -612,6 +1275,7 @@ by creating only one `str` object for repeated map keys. - Fix inaccurate zero padding in serialization of microseconds on `datetime.datetime` objects. + ## 2.0.7 - 2019-08-29 ### Changed @@ -622,12 +1286,14 @@ by creating only one `str` object for repeated map keys. - `orjson.dumps()` raises `JSONEncodeError` on circular references. + ## 2.0.6 - 2019-05-11 ### Changed - Performance improvements. + ## 2.0.5 - 2019-04-19 ### Fixed @@ -636,6 +1302,7 @@ by creating only one `str` object for repeated map keys. 31.245270191439438 was parsed to 31.24527019143944. Serialization was unaffected. + ## 2.0.4 - 2019-04-02 ### Changed @@ -643,24 +1310,28 @@ unaffected. - `orjson.dumps()` now serializes `datetime.datetime` objects without a `tzinfo` rather than raising `JSONEncodeError`. + ## 2.0.3 - 2019-03-23 ### Changed - `orjson.loads()` uses SSE2 to validate `bytes` input. + ## 2.0.2 - 2019-03-12 ### Changed - Support Python 3.5. + ## 2.0.1 - 2019-02-05 ### Changed - Publish Windows wheel. + ## 2.0.0 - 2019-01-28 ### Added @@ -679,30 +1350,35 @@ implementations. - `orjson.dumps()` no longer accepts `bytes`. + ## 1.3.1 - 2019-01-03 ### Fixed - Handle invalid UTF-8 in str. + ## 1.3.0 - 2019-01-02 ### Changed - Performance improvements of 15-25% on serialization, 10% on deserialization. + ## 1.2.1 - 2018-12-31 ### Fixed - Fix memory leak in deserializing dict. + ## 1.2.0 - 2018-12-16 ### Changed - Performance improvements. + ## 1.1.0 - 2018-12-04 ### Changed @@ -713,12 +1389,14 @@ implementations. - Dict key can only be str. + ## 1.0.1 - 2018-11-26 ### Fixed - pyo3 bugfix update. + ## 1.0.0 - 2018-11-23 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 58736726..00000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,6 +0,0 @@ -Please don't open issues asking for support specific to your environment. - -Please don't open an issue asking about something already explained in -the README. - -If you would like to add functionality please first propose it in an issue. \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index db145968..b4f1137a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,252 +1,296 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 - -[[package]] -name = "ahash" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" -dependencies = [ - "getrandom", - "once_cell", - "version_check", -] - -[[package]] -name = "arrayvec" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" -dependencies = [ - "serde", -] +version = 4 [[package]] name = "associative-cache" -version = "1.0.1" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46016233fc1bb55c23b856fe556b7db6ccd05119a0a392e04f0b3b7c79058f16" +checksum = "138b4febdc7d0135523c55358c97361fd45089bc65fe859ef21a58d0892deb00" [[package]] -name = "autocfg" -version = "1.1.0" +name = "bytecount" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" [[package]] -name = "beef" -version = "0.5.2" +name = "bytes" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" -dependencies = [ - "serde", -] +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] -name = "bytecount" -version = "0.6.3" +name = "cc" +version = "1.2.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" +checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" dependencies = [ - "packed_simd_2", + "find-msvc-tools", + "shlex", ] -[[package]] -name = "cc" -version = "1.0.73" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" - [[package]] name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "chrono" -version = "0.4.19" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" -dependencies = [ - "num-integer", - "num-traits", -] +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "encoding_rs" -version = "0.8.31" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if", - "packed_simd_2", ] [[package]] -name = "getrandom" -version = "0.2.7" +name = "find-msvc-tools" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] -name = "inlinable_string" -version = "0.1.15" +name = "gimli" +version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" [[package]] name = "itoa" -version = "1.0.2" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] -name = "libc" -version = "0.2.126" +name = "itoap" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" +checksum = "9028f49264629065d057f340a86acb84867925865f73bbf8d47b4d149a7e88b8" [[package]] -name = "libm" -version = "0.1.4" +name = "jiff" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fc7aa29613bd6a620df431842069224d8bc9011086b1db4c0e0cd47fa03ec9a" +checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" +dependencies = [ + "jiff-static", + "portable-atomic", + "portable-atomic-util", +] [[package]] -name = "num-integer" -version = "0.1.45" +name = "jiff-static" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" dependencies = [ - "autocfg", - "num-traits", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "num-traits" -version = "0.2.15" +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "memchr" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "no-panic" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f967505aabc8af5752d098c34146544a43684817cdba8f9725b292530cabbf53" dependencies = [ - "autocfg", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "once_cell" -version = "1.13.0" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "orjson" -version = "3.7.7" +version = "3.11.9" dependencies = [ - "ahash", - "arrayvec", "associative-cache", - "beef", "bytecount", + "bytes", "cc", - "chrono", "encoding_rs", - "inlinable_string", - "itoa", + "itoap", + "jiff", "once_cell", "pyo3-build-config", "pyo3-ffi", - "ryu", "serde", "serde_json", "simdutf8", - "smallvec", + "unwinding", + "xxhash-rust", + "zmij", ] [[package]] -name = "packed_simd_2" -version = "0.3.8" +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1914cd452d8fccd6f9db48147b29fd4ae05bea9dc5d9ad578509f72415de282" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" dependencies = [ - "cfg-if", - "libm", + "portable-atomic", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", ] [[package]] name = "pyo3-build-config" -version = "0.16.5" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b65b546c35d8a3b1b2f0ddbac7c6a569d759f357f2b9df884f5d6b719152c8" +checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" dependencies = [ - "once_cell", "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.16.5" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c275a07127c1aca33031a563e384ffdd485aee34ef131116fcd58e3430d1742b" +checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" dependencies = [ "libc", "pyo3-build-config", ] [[package]] -name = "ryu" -version = "1.0.10" +name = "quote" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] [[package]] name = "serde" -version = "1.0.138" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1578c6245786b9d168c5447eeacfb96856573ca56c9d68fdcf394be134882a47" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "serde_json" -version = "1.0.82" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "itoa", - "ryu", + "memchr", "serde", + "serde_core", + "zmij", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "simdutf8" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] -name = "smallvec" -version = "1.9.0" +name = "syn" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] [[package]] name = "target-lexicon" -version = "0.12.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02424087780c9b71cc96799eaeddff35af2bc513278cda5c99fc1f5d026d3c1" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] -name = "version_check" -version = "0.9.4" +name = "unicode-ident" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +name = "unwinding" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "60612c845ef41699f39dc8c5391f252942c0a88b7d15da672eff0d14101bbd6d" +dependencies = [ + "gimli", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +dependencies = [ + "no-panic", +] diff --git a/Cargo.toml b/Cargo.toml index 92082992..b05352ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,26 +1,24 @@ [package] name = "orjson" -version = "3.7.7" +version = "3.11.9" authors = ["ijl "] description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" -edition = "2018" -resolver = "2" -rust-version = "1.54" -license = "Apache-2.0 OR MIT" repository = "https://github.com/ijl/orjson" -homepage = "https://github.com/ijl/orjson" -readme = "README.md" +edition = "2024" +resolver = "3" +rust-version = "1.95" +license = "MPL-2.0 AND (Apache-2.0 OR MIT)" keywords = ["fast", "json", "dataclass", "dataclasses", "datetime", "rfc", "8259", "3339"] include = [ "Cargo.toml", "CHANGELOG.md", - "data/*", - "include", + "include/yyjson", "LICENSE-APACHE", "LICENSE-MIT", + "LICENSE-MPL-2.0", "pyproject.toml", "README.md", - "src/*", + "src", "test/*.py", "test/requirements.txt", ] @@ -29,70 +27,53 @@ include = [ name = "orjson" crate-type = ["cdylib"] -[package.metadata.maturin] -requires-python = ">=3.7" -classifiers = [ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "License :: OSI Approved :: MIT License", - "Operating System :: MacOS", - "Operating System :: Microsoft :: Windows", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python", - "Programming Language :: Rust", - "Typing :: Typed", -] - [features] default = [] -# Use SIMD intrinsics. This requires Rust on the nightly channel. -unstable-simd = [ - "bytecount/generic-simd", - "encoding_rs/simd-accel", - "simdutf8/aarch64_neon", -] +# Avoid bundling libgcc on musl. +unwind = ["unwinding"] -# Build yyjson as a backend. -yyjson = [ - "cc", -] +avx512 = [] +generic_simd = [] +inline_int = [] +inline_str = [] +no_panic = ["zmij/no-panic"] +optimize = [] [dependencies] -ahash = { version = "0.7", default_features = false } -arrayvec = { version = "0.7", default_features = false, features = ["std", "serde"] } -associative-cache = { version = "1" } -beef = { version = "0.5", default_features = false, features = ["impl_serde"] } -bytecount = { version = "^0.6.2", default_features = false, features = ["runtime-dispatch-simd"] } -chrono = { version = "0.4", default_features = false } -encoding_rs = { version = "0.8", default_features = false } -inlinable_string = { version = "0.1" } -itoa = { version = "1", default_features = false } -once_cell = { version = "1", default_features = false } -pyo3-ffi = { version = "^0.16.5", default_features = false, features = ["extension-module"]} -ryu = { version = "1", default_features = false } -serde = { version = "1", default_features = false } -serde_json = { version = "^1.0.68", default_features = false, features = ["std", "float_roundtrip"] } -simdutf8 = { version = "0.1", default_features = false, features = ["std"] } -smallvec = { version = "^1.8", default_features = false, features = ["union", "write"] } +associative-cache = { version = "3", default-features = false } +bytecount = { version = "^0.6.7", default-features = false, features = ["runtime-dispatch-simd"] } +bytes = { version = "1", default-features = false } +encoding_rs = { version = "0.8", default-features = false } +itoap = { version = "1", default-features = false, features = ["std", "simd"] } +jiff = { version = "^0.2", default-features = false, features = ["perf-inline"] } +once_cell = { version = "1", default-features = false, features = ["alloc", "race"] } +pyo3-ffi = { version = "0.28", default-features = false } +serde = { version = "1", default-features = false } +serde_json = { version = "1", default-features = false, features = ["std"] } +simdutf8 = { version = "0.1", default-features = false, features = ["std", "public_imp", "aarch64_neon"] } +unwinding = { version = "=0.2.8", default-features = false, features = ["unwinder"], optional = true } +xxhash-rust = { version = "^0.8", default-features = false, features = ["xxh3"] } +zmij = { version = "1", default-features = false } [build-dependencies] -cc = { version = "1", optional = true } -pyo3-build-config = "^0.16.5" +cc = { version = "1" } +pyo3-build-config = { version = "0.28" } + +[profile.dev] +codegen-units = 1 +debug = 2 +debug-assertions = true +incremental = false +lto = "off" +opt-level = 3 +overflow-checks = true [profile.release] codegen-units = 1 debug = false incremental = false -lto = "thin" +lto = "fat" opt-level = 3 panic = "abort" diff --git a/LICENSE-MPL-2.0 b/LICENSE-MPL-2.0 new file mode 100644 index 00000000..ee6256cd --- /dev/null +++ b/LICENSE-MPL-2.0 @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at https://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/README.md b/README.md index 15f3ae8f..2cadb1c6 100644 --- a/README.md +++ b/README.md @@ -1,88 +1,98 @@ # orjson orjson is a fast, correct JSON library for Python. It -[benchmarks](https://github.com/ijl/orjson#performance) as the fastest Python +[benchmarks](https://github.com/ijl/orjson?tab=readme-ov-file#performance) as the fastest Python library for JSON and is more correct than the standard json library or other third-party libraries. It serializes -[dataclass](https://github.com/ijl/orjson#dataclass), -[datetime](https://github.com/ijl/orjson#datetime), -[numpy](https://github.com/ijl/orjson#numpy), and -[UUID](https://github.com/ijl/orjson#uuid) instances natively. - -Its features and drawbacks compared to other Python JSON libraries: - -* serializes `dataclass` instances 40-50x as fast as other libraries -* serializes `datetime`, `date`, and `time` instances to RFC 3339 format, -e.g., "1970-01-01T00:00:00+00:00" -* serializes `numpy.ndarray` instances 4-12x as fast with 0.3x the memory -usage of other libraries -* pretty prints 10x to 20x as fast as the standard library -* serializes to `bytes` rather than `str`, i.e., is not a drop-in replacement -* serializes `str` without escaping unicode to ASCII, e.g., "好" rather than -"\\\u597d" -* serializes `float` 10x as fast and deserializes twice as fast as other -libraries -* serializes subclasses of `str`, `int`, `list`, and `dict` natively, -requiring `default` to specify how to serialize others -* serializes arbitrary types using a `default` hook -* has strict UTF-8 conformance, more correct than the standard library -* has strict JSON conformance in not supporting Nan/Infinity/-Infinity -* has an option for strict JSON conformance on 53-bit integers with default -support for 64-bit -* does not provide `load()` or `dump()` functions for reading from/writing to -file-like objects - -orjson supports CPython 3.7, 3.8, 3.9, 3.10, and 3.11. It distributes x86_64/amd64, -aarch64/armv8, and arm7 wheels for Linux, amd64 and aarch64 wheels for macOS, -and amd64 wheels for Windows. orjson does not support PyPy. Releases -follow semantic versioning and serializing a new object type +[dataclass](https://github.com/ijl/orjson?tab=readme-ov-file#dataclass), +[datetime](https://github.com/ijl/orjson?tab=readme-ov-file#datetime), +[numpy](https://github.com/ijl/orjson?tab=readme-ov-file#numpy), and +[UUID](https://github.com/ijl/orjson?tab=readme-ov-file#uuid) instances natively. + +[orjson.dumps()](https://github.com/ijl/orjson?tab=readme-ov-file#serialize) is +something like 10x as fast as `json`, serializes +common types and subtypes, has a `default` parameter for the caller to specify +how to serialize arbitrary types, and has a number of flags controlling output. + +[orjson.loads()](https://github.com/ijl/orjson?tab=readme-ov-file#deserialize) +is something like 2x as fast as `json`, and is strictly compliant with UTF-8 and +RFC 8259 ("The JavaScript Object Notation (JSON) Data Interchange Format"). + +Reading from and writing to files, line-delimited JSON files, and so on is +not provided by the library. + +orjson supports CPython 3.10, 3.11, 3.12, 3.13, 3.14, and 3.15. + +It distributes amd64/x86_64/x64, i686/x86, aarch64/arm64/armv8, arm7, +ppc64le/POWER8, and s390x wheels for Linux, amd64 and aarch64 wheels +for macOS, and amd64, i686, and aarch64 wheels for Windows. + +Wheels published to PyPI for amd64 run on x86-64-v1 (2003) +or later, but will at runtime use AVX-512 if available for a +significant performance benefit; aarch64 wheels run on ARMv8-A (2011) or +later. + +orjson does not and will not support PyPy, embedded Python builds for +Android/iOS, or PEP 554 subinterpreters. + +orjson may support PEP 703 free-threading when it is stable. + +Releases follow semantic versioning and serializing a new object type without an opt-in flag is considered a breaking change. -orjson is licensed under both the Apache 2.0 and MIT licenses. The -repository and issue tracker is -[github.com/ijl/orjson](https://github.com/ijl/orjson), and patches may be -submitted there. There is a -[CHANGELOG](https://github.com/ijl/orjson/blob/master/CHANGELOG.md) +orjson contains source code licensed under the Mozilla Public License 2.0, +Apache 2.0, and MIT licenses. The repository from which PyPI artifacts are +published is [github.com/ijl/orjson](https://github.com/ijl/orjson) and an +alternative repository is [codeberg.org/ijl/orjson](https://codeberg.org/ijl/orjson). +There is no open issue tracker or pull requests due to signal-to-noise ratio. +There is a [CHANGELOG](https://github.com/ijl/orjson/blob/master/CHANGELOG.md) available in the repository. -1. [Usage](https://github.com/ijl/orjson#usage) - 1. [Install](https://github.com/ijl/orjson#install) - 2. [Quickstart](https://github.com/ijl/orjson#quickstart) - 3. [Migrating](https://github.com/ijl/orjson#migrating) - 4. [Serialize](https://github.com/ijl/orjson#serialize) - 1. [default](https://github.com/ijl/orjson#default) - 2. [option](https://github.com/ijl/orjson#option) - 5. [Deserialize](https://github.com/ijl/orjson#deserialize) -2. [Types](https://github.com/ijl/orjson#types) - 1. [dataclass](https://github.com/ijl/orjson#dataclass) - 2. [datetime](https://github.com/ijl/orjson#datetime) - 3. [enum](https://github.com/ijl/orjson#enum) - 4. [float](https://github.com/ijl/orjson#float) - 5. [int](https://github.com/ijl/orjson#int) - 6. [numpy](https://github.com/ijl/orjson#numpy) - 7. [str](https://github.com/ijl/orjson#str) - 8. [uuid](https://github.com/ijl/orjson#uuid) -3. [Testing](https://github.com/ijl/orjson#testing) -4. [Performance](https://github.com/ijl/orjson#performance) - 1. [Latency](https://github.com/ijl/orjson#latency) - 2. [Memory](https://github.com/ijl/orjson#memory) - 3. [Reproducing](https://github.com/ijl/orjson#reproducing) -5. [Questions](https://github.com/ijl/orjson#questions) -6. [Packaging](https://github.com/ijl/orjson#packaging) -7. [License](https://github.com/ijl/orjson#license) +1. [Usage](https://github.com/ijl/orjson?tab=readme-ov-file#usage) + 1. [Install](https://github.com/ijl/orjson?tab=readme-ov-file#install) + 2. [Quickstart](https://github.com/ijl/orjson?tab=readme-ov-file#quickstart) + 3. [Migrating](https://github.com/ijl/orjson?tab=readme-ov-file#migrating) + 4. [Serialize](https://github.com/ijl/orjson?tab=readme-ov-file#serialize) + 1. [default](https://github.com/ijl/orjson?tab=readme-ov-file#default) + 2. [option](https://github.com/ijl/orjson?tab=readme-ov-file#option) + 3. [Fragment](https://github.com/ijl/orjson?tab=readme-ov-file#fragment) + 5. [Deserialize](https://github.com/ijl/orjson?tab=readme-ov-file#deserialize) +2. [Types](https://github.com/ijl/orjson?tab=readme-ov-file#types) + 1. [dataclass](https://github.com/ijl/orjson?tab=readme-ov-file#dataclass) + 2. [datetime](https://github.com/ijl/orjson?tab=readme-ov-file#datetime) + 3. [enum](https://github.com/ijl/orjson?tab=readme-ov-file#enum) + 4. [float](https://github.com/ijl/orjson?tab=readme-ov-file#float) + 5. [int](https://github.com/ijl/orjson?tab=readme-ov-file#int) + 6. [numpy](https://github.com/ijl/orjson?tab=readme-ov-file#numpy) + 7. [str](https://github.com/ijl/orjson?tab=readme-ov-file#str) + 8. [uuid](https://github.com/ijl/orjson?tab=readme-ov-file#uuid) +3. [Testing](https://github.com/ijl/orjson?tab=readme-ov-file#testing) +4. [Performance](https://github.com/ijl/orjson?tab=readme-ov-file#performance) + 1. [Latency](https://github.com/ijl/orjson?tab=readme-ov-file#latency) + 2. [Reproducing](https://github.com/ijl/orjson?tab=readme-ov-file#reproducing) +5. [Questions](https://github.com/ijl/orjson?tab=readme-ov-file#questions) +6. [Packaging](https://github.com/ijl/orjson?tab=readme-ov-file#packaging) +7. [License](https://github.com/ijl/orjson?tab=readme-ov-file#license) ## Usage ### Install -To install a wheel from PyPI: +To install a wheel from PyPI, install the `orjson` package. -```sh -pip install --upgrade "pip>=20.3" # manylinux_x_y, universal2 wheel support -pip install --upgrade orjson +In `requirements.in` or `requirements.txt` format, specify: + +```txt +orjson >= 3.10,<4 +``` + +In `pyproject.toml` format, specify: + +```toml +orjson = "^3.10" ``` -To build a wheel, see [packaging](https://github.com/ijl/orjson#packaging). +To build a wheel, see [packaging](https://github.com/ijl/orjson?tab=readme-ov-file#packaging). ### Quickstart @@ -116,13 +126,18 @@ implementations in a `default` function and options enabling them can be removed but do not need to be. There was no change in deserialization. To migrate from the standard library, the largest difference is that -`orjson.dumps` returns `bytes` and `json.dumps` returns a `str`. Users with -`dict` objects using non-`str` keys should specify -`option=orjson.OPT_NON_STR_KEYS`. `sort_keys` is replaced by -`option=orjson.OPT_SORT_KEYS`. `indent` is replaced by -`option=orjson.OPT_INDENT_2` and other levels of indentation are not +`orjson.dumps` returns `bytes` and `json.dumps` returns a `str`. + +Users with `dict` objects using non-`str` keys should specify `option=orjson.OPT_NON_STR_KEYS`. + +`sort_keys` is replaced by `option=orjson.OPT_SORT_KEYS`. + +`indent` is replaced by `option=orjson.OPT_INDENT_2` and other levels of indentation are not supported. +`ensure_ascii` is probably not relevant today and UTF-8 characters cannot be +escaped to ASCII. + ### Serialize ```python @@ -136,10 +151,10 @@ def dumps( `dumps()` serializes Python objects to JSON. It natively serializes -`str`, `dict`, `list`, `tuple`, `int`, `float`, `bool`, +`str`, `dict`, `list`, `tuple`, `int`, `float`, `bool`, `None`, `dataclasses.dataclass`, `typing.TypedDict`, `datetime.datetime`, `datetime.date`, `datetime.time`, `uuid.UUID`, `numpy.ndarray`, and -`None` instances. It supports arbitrary types through `default`. It +`orjson.Fragment` instances. It supports arbitrary types through `default`. It serializes subclasses of `str`, `int`, `dict`, `list`, `dataclasses.dataclass`, and `enum.Enum`. It does not serialize subclasses of `tuple` to avoid serializing `namedtuple` objects as arrays. To avoid @@ -152,7 +167,7 @@ The global interpreter lock (GIL) is held for the duration of the call. It raises `JSONEncodeError` on an unsupported type. This exception message describes the invalid object with the error message `Type is not JSON serializable: ...`. To fix this, specify -[default](https://github.com/ijl/orjson#default). +[default](https://github.com/ijl/orjson?tab=readme-ov-file#default). It raises `JSONEncodeError` on a `str` that contains invalid UTF-8. @@ -173,6 +188,9 @@ unsupported. `JSONEncodeError` is a subclass of `TypeError`. This is for compatibility with the standard library. +If the failure was caused by an exception in `default` then +`JSONEncodeError` chains the original exception as `__cause__`. + #### default To serialize a subclass or arbitrary types, specify `default` as a @@ -205,7 +223,7 @@ Python otherwise implicitly returns `None`, which appears to the caller like a legitimate value and is serialized: ```python ->>> import orjson, json, rapidjson +>>> import orjson, json >>> def default(obj): if isinstance(obj, decimal.Decimal): @@ -215,8 +233,6 @@ def default(obj): b'{"set":null}' >>> json.dumps({"set":{1, 2}}, default=default) '{"set":null}' ->>> rapidjson.dumps({"set":{1, 2}}, default=default) -'{"set":null}' ``` #### option @@ -243,9 +259,7 @@ b"[]\n" Pretty-print output with an indent of two spaces. This is equivalent to `indent=2` in the standard library. Pretty printing is slower and the output -larger. orjson is the fastest compared library at pretty printing and has -much less of a slowdown to pretty print than the standard library does. This -option is compatible with all other options. +larger. This option is compatible with all other options. ```python >>> import orjson @@ -276,28 +290,21 @@ If displayed, the indentation and linebreaks appear like this: This measures serializing the github.json fixture as compact (52KiB) or pretty (64KiB): -| Library | compact (ms) | pretty (ms) | vs. orjson | -|------------|----------------|---------------|--------------| -| orjson | 0.06 | 0.07 | 1.0 | -| ujson | 0.18 | 0.19 | 2.8 | -| rapidjson | 0.22 | | | -| simplejson | 0.35 | 1.49 | 21.4 | -| json | 0.36 | 1.19 | 17.2 | +| Library | compact (ms) | pretty (ms) | vs. orjson | +|-----------|----------------|---------------|--------------| +| orjson | 0.01 | 0.02 | 1 | +| json | 0.13 | 0.54 | 34 | This measures serializing the citm_catalog.json fixture, more of a worst case due to the amount of nesting and newlines, as compact (489KiB) or pretty (1.1MiB): -| Library | compact (ms) | pretty (ms) | vs. orjson | -|------------|----------------|---------------|--------------| -| orjson | 0.88 | 1.73 | 1.0 | -| ujson | 3.73 | 4.52 | 2.6 | -| rapidjson | 3.54 | | | -| simplejson | 11.77 | 72.06 | 41.6 | -| json | 6.71 | 55.22 | 31.9 | +| Library | compact (ms) | pretty (ms) | vs. orjson | +|-----------|----------------|---------------|--------------| +| orjson | 0.25 | 0.45 | 1 | +| json | 3.01 | 24.42 | 54.4 | -rapidjson is blank because it does not support pretty printing. This can be -reproduced using the `pyindent` script. +This can be reproduced using the `pyindent` script. ##### OPT_NAIVE_UTC @@ -370,18 +377,14 @@ single integer. In "str keys", the keys were converted to `str` before serialization, and orjson still specifes `option=orjson.OPT_NON_STR_KEYS` (which is always somewhat slower). -| Library | str keys (ms) | int keys (ms) | int keys sorted (ms) | -|------------|-----------------|-----------------|------------------------| -| orjson | 1.53 | 2.16 | 4.29 | -| ujson | 3.07 | 5.65 | | -| rapidjson | 4.29 | | | -| simplejson | 11.24 | 14.50 | 21.86 | -| json | 7.17 | 8.49 | | +| Library | str keys (ms) | int keys (ms) | int keys sorted (ms) | +|-----------|-----------------|-----------------|------------------------| +| orjson | 0.5 | 0.93 | 2.08 | +| json | 2.72 | 3.59 | | -ujson is blank for sorting because it segfaults. json is blank because it +json is blank because it raises `TypeError` on attempting to sort before converting all keys to `str`. -rapidjson is blank because it does not support non-`str` keys. This can -be reproduced using the `pynonstr` script. +This can be reproduced using the `pynonstr` script. ##### OPT_OMIT_MICROSECONDS @@ -491,18 +494,18 @@ OPT_NON_STR_KEYS. This is deprecated and has no effect in version 3. In version 2 this was required to serialize `dataclasses.dataclass` instances. For more, see -[dataclass](https://github.com/ijl/orjson#dataclass). +[dataclass](https://github.com/ijl/orjson?tab=readme-ov-file#dataclass). ##### OPT_SERIALIZE_NUMPY Serialize `numpy.ndarray` instances. For more, see -[numpy](https://github.com/ijl/orjson#numpy). +[numpy](https://github.com/ijl/orjson?tab=readme-ov-file#numpy). ##### OPT_SERIALIZE_UUID This is deprecated and has no effect in version 3. In version 2 this was required to serialize `uuid.UUID` instances. For more, see -[UUID](https://github.com/ijl/orjson#UUID). +[UUID](https://github.com/ijl/orjson?tab=readme-ov-file#UUID). ##### OPT_SORT_KEYS @@ -523,13 +526,10 @@ b'{"a":3,"b":1,"c":2}' This measures serializing the twitter.json fixture unsorted and sorted: -| Library | unsorted (ms) | sorted (ms) | vs. orjson | -|------------|-----------------|---------------|--------------| -| orjson | 0.5 | 0.92 | 1 | -| ujson | 1.61 | 2.48 | 2.7 | -| rapidjson | 2.17 | 2.89 | 3.2 | -| simplejson | 3.56 | 5.13 | 5.6 | -| json | 3.59 | 4.59 | 5 | +| Library | unsorted (ms) | sorted (ms) | vs. orjson | +|-----------|-----------------|---------------|--------------| +| orjson | 0.11 | 0.3 | 1 | +| json | 1.36 | 1.93 | 6.4 | The benchmark can be reproduced using the `pysort` script. @@ -541,15 +541,14 @@ The sorting is not collation/locale-aware: b'{"A":3,"a":1,"\xc3\xa4":2}' ``` -This is the same sorting behavior as the standard library, rapidjson, -simplejson, and ujson. +This is the same sorting behavior as the standard library. `dataclass` also serialize as maps but this has no effect on them. ##### OPT_STRICT_INTEGER Enforce 53-bit limit on integers. The limit is otherwise 64 bits, the same as -the Python standard library. For more, see [int](https://github.com/ijl/orjson#int). +the Python standard library. For more, see [int](https://github.com/ijl/orjson?tab=readme-ov-file#int). ##### OPT_UTC_Z @@ -569,6 +568,28 @@ b'"1970-01-01T00:00:00+00:00"' b'"1970-01-01T00:00:00Z"' ``` +#### Fragment + +`orjson.Fragment` includes already-serialized JSON in a document. This is an +efficient way to include JSON blobs from a cache, JSONB field, or separately +serialized object without first deserializing to Python objects via `loads()`. + +```python +>>> import orjson +>>> orjson.dumps({"key": "zxc", "data": orjson.Fragment(b'{"a": "b", "c": 1}')}) +b'{"key":"zxc","data":{"a": "b", "c": 1}}' +``` + +It does no reformatting: `orjson.OPT_INDENT_2` will not affect a +compact blob nor will a pretty-printed JSON blob be rewritten as compact. + +The input must be `bytes` or `str` and given as a positional argument. + +This raises `orjson.JSONEncodeError` if a `str` is given and the input is +not valid UTF-8. It otherwise does no validation and it is possible to +write invalid JSON. This does not escape characters. The implementation is +tested to not crash if given invalid strings or invalid JSON. + ### Deserialize ```python @@ -580,14 +601,15 @@ def loads(__obj: Union[bytes, bytearray, memoryview, str]) -> Any: ... `bytes`, `bytearray`, `memoryview`, and `str` input are accepted. If the input exists as a `memoryview`, `bytearray`, or `bytes` object, it is recommended to -pass these directly rather than creating an unnecessary `str` object. This has -lower memory usage and lower latency. +pass these directly rather than creating an unnecessary `str` object. That is, +`orjson.loads(b"{}")` instead of `orjson.loads(b"{}".decode("utf-8"))`. This +has lower memory usage and lower latency. The input must be valid UTF-8. orjson maintains a cache of map keys for the duration of the process. This causes a net reduction in memory usage by avoiding duplicate strings. The -keys must be at most 64 bytes to be cached and 512 entries are stored. +keys must be at most 64 bytes to be cached and 2048 entries are stored. The global interpreter lock (GIL) is held for the duration of the call. @@ -595,6 +617,12 @@ It raises `JSONDecodeError` if given an invalid type or invalid JSON. This includes if the input contains `NaN`, `Infinity`, or `-Infinity`, which the standard library allows, but is not valid JSON. +It raises `JSONDecodeError` if a combination of array or object recurses +1024 levels deep. + +It raises `JSONDecodeError` if unable to allocate a buffer large enough +to parse the document. + `JSONDecodeError` is a subclass of `json.JSONDecodeError` and `ValueError`. This is for compatibility with the standard library. @@ -611,13 +639,10 @@ using `__slots__`, frozen dataclasses, those with optional or default attributes, and subclasses. There is a performance benefit to not using `__slots__`. -| Library | dict (ms) | dataclass (ms) | vs. orjson | -|------------|-------------|------------------|--------------| -| orjson | 1.40 | 1.60 | 1 | -| ujson | | | | -| rapidjson | 3.64 | 68.48 | 42 | -| simplejson | 14.21 | 92.18 | 57 | -| json | 13.28 | 94.90 | 59 | +| Library | dict (ms) | dataclass (ms) | vs. orjson | +|-----------|-------------|------------------|--------------| +| orjson | 0.43 | 0.95 | 1 | +| json | 5.81 | 38.32 | 40 | This measures serializing 555KiB of JSON, orjson natively and other libraries using `default` to serialize the output of `dataclasses.asdict()`. This can be @@ -668,7 +693,7 @@ b'"2100-09-01T21:55:02"' ``` `datetime.datetime` supports instances with a `tzinfo` that is `None`, -`datetime.timezone.utc`, a timezone instance from the python3.9+ `zoneinfo` +`datetime.timezone.utc`, a timezone instance from the standard library `zoneinfo` module, or a timezone instance from the third-party `pendulum`, `pytz`, or `dateutil`/`arrow` libraries. @@ -692,10 +717,6 @@ b'"1900-01-02"' Errors with `tzinfo` result in `JSONEncodeError` being raised. -It is faster to have orjson serialize datetime objects than to do so -before calling `dumps()`. If using an unsupported type such as -`pendulum.datetime`, use `default`. - To disable serialization of `datetime` objects specify the option `orjson.OPT_PASSTHROUGH_DATETIME`. @@ -750,13 +771,9 @@ precision and consistent rounding. compliant JSON, as `null`: ```python ->>> import orjson, ujson, rapidjson, json +>>> import orjson, json >>> orjson.dumps([float("NaN"), float("Infinity"), float("-Infinity")]) b'[null,null,null]' ->>> ujson.dumps([float("NaN"), float("Infinity"), float("-Infinity")]) -OverflowError: Invalid Inf value when encoding double ->>> rapidjson.dumps([float("NaN"), float("Infinity"), float("-Infinity")]) -'[NaN,Infinity,-Infinity]' >>> json.dumps([float("NaN"), float("Infinity"), float("-Infinity")]) '[NaN, Infinity, -Infinity]' ``` @@ -783,10 +800,14 @@ JSONEncodeError: Integer exceeds 53-bit range ### numpy -orjson natively serializes `numpy.ndarray` and individual `numpy.float64`, -`numpy.float32`, `numpy.int64`, `numpy.int32`, `numpy.int8`, `numpy.uint64`, -`numpy.uint32`, `numpy.uint8`, `numpy.uintp`, or `numpy.intp`, and -`numpy.datetime64` instances. +orjson natively serializes `numpy.ndarray` and individual +`numpy.float64`, `numpy.float32`, `numpy.float16` (`numpy.half`), +`numpy.int64`, `numpy.int32`, `numpy.int16`, `numpy.int8`, +`numpy.uint64`, `numpy.uint32`, `numpy.uint16`, `numpy.uint8`, +`numpy.uintp`, `numpy.intp`, `numpy.datetime64`, and `numpy.bool` +instances. + +orjson is compatible with both numpy v1 and v2. orjson is faster than all compared libraries at serializing numpy instances. Serializing numpy data requires specifying @@ -804,6 +825,11 @@ b'[[1,2,3],[4,5,6]]' The array must be a contiguous C array (`C_CONTIGUOUS`) and one of the supported datatypes. +Note a difference between serializing `numpy.float32` using `ndarray.tolist()` +or `orjson.dumps(..., option=orjson.OPT_SERIALIZE_NUMPY)`: `tolist()` converts +to a `double` before serializing and orjson's native path does not. This +can result in different rounding. + `numpy.datetime64` instances are serialized as RFC 3339 strings and datetime options affect them. @@ -825,47 +851,41 @@ b'"2021-01-01T00:00:00.172000"' b'"2021-01-01T00:00:00+00:00"' ``` -If an array is not a contiguous C array, contains an supported datatype, +If an array is not a contiguous C array, contains an unsupported datatype, or contains a `numpy.datetime64` using an unsupported representation (e.g., picoseconds), orjson falls through to `default`. In `default`, -`obj.tolist()` can be specified. If an array is malformed, which -is not expected, `orjson.JSONEncodeError` is raised. +`obj.tolist()` can be specified. + +If an array is not in the native endianness, e.g., an array of big-endian values +on a little-endian system, `orjson.JSONEncodeError` is raised. + +If an array is malformed, `orjson.JSONEncodeError` is raised. This measures serializing 92MiB of JSON from an `numpy.ndarray` with dimensions of `(50000, 100)` and `numpy.float64` values: -| Library | Latency (ms) | RSS diff (MiB) | vs. orjson | -|------------|----------------|------------------|--------------| -| orjson | 194 | 99 | 1.0 | -| ujson | | | | -| rapidjson | 3,048 | 309 | 15.7 | -| simplejson | 3,023 | 297 | 15.6 | -| json | 3,133 | 297 | 16.1 | +| Library | Latency (ms) | RSS diff (MiB) | vs. orjson | +|-----------|----------------|------------------|--------------| +| orjson | 105 | 105 | 1 | +| json | 1,481 | 295 | 14.2 | This measures serializing 100MiB of JSON from an `numpy.ndarray` with dimensions of `(100000, 100)` and `numpy.int32` values: -| Library | Latency (ms) | RSS diff (MiB) | vs. orjson | -|------------|----------------|------------------|--------------| -| orjson | 178 | 115 | 1.0 | -| ujson | | | | -| rapidjson | 1,512 | 551 | 8.5 | -| simplejson | 1,606 | 504 | 9.0 | -| json | 1,506 | 503 | 8.4 | +| Library | Latency (ms) | RSS diff (MiB) | vs. orjson | +|-----------|----------------|------------------|--------------| +| orjson | 68 | 119 | 1 | +| json | 684 | 501 | 10.1 | This measures serializing 105MiB of JSON from an `numpy.ndarray` with dimensions of `(100000, 200)` and `numpy.bool` values: -| Library | Latency (ms) | RSS diff (MiB) | vs. orjson | -|------------|----------------|------------------|--------------| -| orjson | 157 | 120 | 1.0 | -| ujson | | | | -| rapidjson | 710 | 327 | 4.5 | -| simplejson | 931 | 398 | 5.9 | -| json | 996 | 400 | 6.3 | +| Library | Latency (ms) | RSS diff (MiB) | vs. orjson | +|-----------|----------------|------------------|--------------| +| orjson | 50 | 125 | 1 | +| json | 573 | 398 | 11.5 | -In these benchmarks, orjson serializes natively, ujson is blank because it -does not support a `default` parameter, and the other libraries serialize +In these benchmarks, orjson serializes natively and `json` serializes `ndarray.tolist()` via `default`. The RSS column measures peak memory usage during serialization. This can be reproduced using the `pynumpy` script. @@ -883,25 +903,14 @@ If `orjson.dumps()` is given a `str` that does not contain valid UTF-8, `orjson.JSONEncodeError` is raised. If `loads()` receives invalid UTF-8, `orjson.JSONDecodeError` is raised. -orjson and rapidjson are the only compared JSON libraries to consistently -error on bad input. - ```python ->>> import orjson, ujson, rapidjson, json +>>> import orjson, json >>> orjson.dumps('\ud800') JSONEncodeError: str is not valid UTF-8: surrogates not allowed ->>> ujson.dumps('\ud800') -UnicodeEncodeError: 'utf-8' codec ... ->>> rapidjson.dumps('\ud800') -UnicodeEncodeError: 'utf-8' codec ... >>> json.dumps('\ud800') '"\\ud800"' >>> orjson.loads('"\\ud800"') JSONDecodeError: unexpected end of hex escape at line 1 column 8: line 1 column 1 (char 0) ->>> ujson.loads('"\\ud800"') -'' ->>> rapidjson.loads('"\\ud800"') -ValueError: Parse error at offset 1: The surrogate pair in string is invalid. >>> json.loads('"\\ud800"') '\ud800' ``` @@ -925,8 +934,6 @@ orjson serializes `uuid.UUID` instances to ``` python >>> import orjson, uuid ->>> orjson.dumps(uuid.UUID('f81d4fae-7dec-11d0-a765-00a0c91e6bf6')) -b'"f81d4fae-7dec-11d0-a765-00a0c91e6bf6"' >>> orjson.dumps(uuid.uuid5(uuid.NAMESPACE_DNS, "python.org")) b'"886313e1-3b8a-5372-9b90-0c9aee199e5d"' ``` @@ -941,8 +948,7 @@ repositories. It is tested to not crash against the It is tested to not leak memory. It is tested to not crash against and not accept invalid UTF-8. There are integration tests exercising the library's use in web servers (gunicorn using multiprocess/forked -workers) and when -multithreaded. It also uses some tests from the ultrajson library. +workers) and when multithreaded. orjson is the most correct of the compared libraries. This graph shows how each library handles a combined 342 JSON fixtures from the @@ -952,9 +958,6 @@ library handles a combined 342 JSON fixtures from the | Library | Invalid JSON documents not rejected | Valid JSON documents not deserialized | |------------|---------------------------------------|-----------------------------------------| | orjson | 0 | 0 | -| ujson | 38 | 0 | -| rapidjson | 6 | 0 | -| simplejson | 13 | 0 | | json | 17 | 0 | This shows that all libraries deserialize valid JSON but only orjson @@ -965,170 +968,81 @@ The graph above can be reproduced using the `pycorrectness` script. ## Performance -Serialization and deserialization performance of orjson is better than -ultrajson, rapidjson, simplejson, or json. The benchmarks are done on -fixtures of real data: - -* twitter.json, 631.5KiB, results of a search on Twitter for "一", containing -CJK strings, dictionaries of strings and arrays of dictionaries, indented. - -* github.json, 55.8KiB, a GitHub activity feed, containing dictionaries of -strings and arrays of dictionaries, not indented. +Serialization and deserialization performance of orjson is consistently better +than the standard library's `json`. The graphs below illustrate a few commonly +used documents. -* citm_catalog.json, 1.7MiB, concert data, containing nested dictionaries of -strings and arrays of integers, indented. +### Latency -* canada.json, 2.2MiB, coordinates of the Canadian border in GeoJSON -format, containing floats and arrays, indented. +![Serialization](doc/serialization.png) -### Latency +![Deserialization](doc/deserialization.png) #### twitter.json serialization -| Library | Median latency (milliseconds) | Operations per second | Relative (latency) | -|------------|---------------------------------|-------------------------|----------------------| -| orjson | 0.41 | 2419.7 | 1 | -| ujson | 1.8 | 555.2 | 4.36 | -| rapidjson | 1.26 | 795 | 3.05 | -| simplejson | 2.27 | 440.6 | 5.5 | -| json | 1.83 | 548.2 | 4.42 | +| Library | Median latency (milliseconds) | Operations per second | Relative (latency) | +|-----------|---------------------------------|-------------------------|----------------------| +| orjson | 0.1 | 8453 | 1 | +| json | 1.3 | 765 | 11.1 | #### twitter.json deserialization -| Library | Median latency (milliseconds) | Operations per second | Relative (latency) | -|------------|---------------------------------|-------------------------|----------------------| -| orjson | 0.85 | 1173 | 1 | -| ujson | 1.88 | 532.1 | 2.2 | -| rapidjson | 2.7 | 371 | 3.16 | -| simplejson | 2.16 | 463.1 | 2.53 | -| json | 2.33 | 429.7 | 2.73 | +| Library | Median latency (milliseconds) | Operations per second | Relative (latency) | +|-----------|---------------------------------|-------------------------|----------------------| +| orjson | 0.5 | 1889 | 1 | +| json | 2.2 | 453 | 4.2 | #### github.json serialization -| Library | Median latency (milliseconds) | Operations per second | Relative (latency) | -|------------|---------------------------------|-------------------------|----------------------| -| orjson | 0.04 | 23751.2 | 1 | -| ujson | 0.18 | 5498.1 | 4.31 | -| rapidjson | 0.1 | 9557 | 2.48 | -| simplejson | 0.25 | 3989.7 | 5.94 | -| json | 0.18 | 5457.6 | 4.36 | +| Library | Median latency (milliseconds) | Operations per second | Relative (latency) | +|-----------|---------------------------------|-------------------------|----------------------| +| orjson | 0.01 | 103693 | 1 | +| json | 0.13 | 7648 | 13.6 | #### github.json deserialization -| Library | Median latency (milliseconds) | Operations per second | Relative (latency) | -|------------|---------------------------------|-------------------------|----------------------| -| orjson | 0.07 | 14680.6 | 1 | -| ujson | 0.19 | 5224.3 | 2.81 | -| rapidjson | 0.17 | 5913.2 | 2.49 | -| simplejson | 0.15 | 6840.8 | 2.15 | -| json | 0.15 | 6480.2 | 2.27 | +| Library | Median latency (milliseconds) | Operations per second | Relative (latency) | +|-----------|---------------------------------|-------------------------|----------------------| +| orjson | 0.04 | 23264 | 1 | +| json | 0.1 | 10430 | 2.2 | #### citm_catalog.json serialization -| Library | Median latency (milliseconds) | Operations per second | Relative (latency) | -|------------|---------------------------------|-------------------------|----------------------| -| orjson | 0.7 | 1420.8 | 1 | -| ujson | 2.89 | 345.2 | 4.1 | -| rapidjson | 1.84 | 543.3 | 2.61 | -| simplejson | 10.06 | 99.4 | 14.29 | -| json | 3.94 | 254 | 5.59 | +| Library | Median latency (milliseconds) | Operations per second | Relative (latency) | +|-----------|---------------------------------|-------------------------|----------------------| +| orjson | 0.3 | 3975 | 1 | +| json | 3 | 338 | 11.8 | #### citm_catalog.json deserialization -| Library | Median latency (milliseconds) | Operations per second | Relative (latency) | -|------------|---------------------------------|-------------------------|----------------------| -| orjson | 1.72 | 579.6 | 1 | -| ujson | 3.68 | 272.1 | 2.13 | -| rapidjson | 5.61 | 178.4 | 3.26 | -| simplejson | 5.06 | 198.2 | 2.94 | -| json | 5.09 | 196.9 | 2.95 | +| Library | Median latency (milliseconds) | Operations per second | Relative (latency) | +|-----------|---------------------------------|-------------------------|----------------------| +| orjson | 1.3 | 781 | 1 | +| json | 4 | 250 | 3.1 | #### canada.json serialization -| Library | Median latency (milliseconds) | Operations per second | Relative (latency) | -|------------|---------------------------------|-------------------------|----------------------| -| orjson | 3.65 | 274.5 | 1 | -| ujson | 12.59 | 79.3 | 3.45 | -| rapidjson | 34.24 | 29.2 | 9.39 | -| simplejson | 57.43 | 17.4 | 15.75 | -| json | 36.03 | 27.6 | 9.88 | +| Library | Median latency (milliseconds) | Operations per second | Relative (latency) | +|-----------|---------------------------------|-------------------------|----------------------| +| orjson | 2.5 | 399 | 1 | +| json | 29.8 | 33 | 11.9 | #### canada.json deserialization -| Library | Median latency (milliseconds) | Operations per second | Relative (latency) | -|------------|---------------------------------|-------------------------|----------------------| -| orjson | 4.18 | 240.2 | 1 | -| ujson | 9.29 | 107.8 | 2.22 | -| rapidjson | 23.56 | 42.4 | 5.64 | -| simplejson | 21.93 | 45.5 | 5.25 | -| json | 21.34 | 46.9 | 5.11 | - -### Memory - -orjson as of 3.7.0 has higher baseline memory usage than other libraries -due to a persistent buffer used for parsing. Incremental memory usage when -deserializing is similar to the standard library and other third-party -libraries. - -This measures, in the first column, RSS after importing a library and reading -the fixture, and in the second column, increases in RSS after repeatedly -calling `loads()` on the fixture. - -#### twitter.json - -| Library | import, read() RSS (MiB) | loads() increase in RSS (MiB) | -|------------|----------------------------|---------------------------------| -| orjson | 21.8 | 2.8 | -| ujson | 14.3 | 4.8 | -| rapidjson | 14.9 | 4.6 | -| simplejson | 13.4 | 2.4 | -| json | 13.1 | 2.3 | - -#### github.json - -| Library | import, read() RSS (MiB) | loads() increase in RSS (MiB) | -|------------|----------------------------|---------------------------------| -| orjson | 21.2 | 0.5 | -| ujson | 13.6 | 0.6 | -| rapidjson | 14.1 | 0.5 | -| simplejson | 12.5 | 0.3 | -| json | 12.4 | 0.3 | - -#### citm_catalog.json - -| Library | import, read() RSS (MiB) | loads() increase in RSS (MiB) | -|------------|----------------------------|---------------------------------| -| orjson | 23 | 10.6 | -| ujson | 15.2 | 11.2 | -| rapidjson | 15.8 | 29.7 | -| simplejson | 14.4 | 24.7 | -| json | 13.9 | 24.7 | - -#### canada.json - -| Library | import, read() RSS (MiB) | loads() increase in RSS (MiB) | -|------------|----------------------------|---------------------------------| -| orjson | 23.2 | 21.3 | -| ujson | 15.6 | 19.2 | -| rapidjson | 16.3 | 23.4 | -| simplejson | 15 | 21.1 | -| json | 14.3 | 20.9 | +| Library | Median latency (milliseconds) | Operations per second | Relative (latency) | +|-----------|---------------------------------|-------------------------|----------------------| +| orjson | 3 | 333 | 1 | +| json | 18 | 55 | 6 | ### Reproducing -The above was measured using Python 3.10.4 on Linux (amd64) with -orjson 3.7.0, ujson 5.3.0, python-rapidson 1.6, and simplejson 3.17.6. - -The latency results can be reproduced using the `pybench` and `graph` -scripts. The memory results can be reproduced using the `pymem` script. +The above was measured using Python 3.11.10 in a Fedora 42 container on an +x86-64-v4 machine using the +`orjson-3.10.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl` +artifact on PyPI. The latency results can be reproduced using the `pybench` script. ## Questions -### Why can't I install it from PyPI? - -Probably `pip` needs to be upgraded to version 20.3 or later to support -the latest manylinux_x_y or universal2 wheel formats. - ### Will it deserialize to dataclasses, UUIDs, decimals, etc or support object_hook? No. This requires a schema specifying what types are expected and how to @@ -1139,43 +1053,54 @@ level above this. No. `bytes` is the correct type for a serialized blob. -### Will it support PyPy? +### Will it support NDJSON or JSONL? + +No. [orjsonl](https://github.com/umarbutler/orjsonl) may be appropriate. -Probably not. +### Will it support JSON5 or RJSON? + +No, it supports RFC 8259. + +### How do I depend on orjson in a Rust project? + +orjson is only shipped as a Python module. The project should depend on +`orjson` in its own Python requirements and should obtain pointers to +functions and objects using the normal `PyImport_*` APIs. ## Packaging -To package orjson requires at least [Rust](https://www.rust-lang.org/) 1.54 -and the [maturin](https://github.com/PyO3/maturin) build tool. It benefits -from also having `clang`. The recommended build command is: +To package orjson requires at least [Rust](https://www.rust-lang.org/) 1.95, +a C compiler, and the [maturin](https://github.com/PyO3/maturin) build tool. +The recommended build command is: ```sh -maturin build --no-sdist --release --strip --cargo-extra-args="--features=yyjson" +maturin build --release --strip ``` -To build without use of `clang`, do not specify `--features=yyjson`. -Deserialization is much faster if built with this feature. - -There is a minor performance benefit on at least amd64 to building on `nightly` -with `--features=unstable-simd`. It may be more significant on other -architectures. - -The project's own CI tests against `nightly-2022-06-22` and stable 1.54. It +The project's own CI tests against `nightly-2026-05-01` and stable 1.95. It is prudent to pin the nightly version because that channel can introduce -breaking changes. +breaking changes. There is a significant performance benefit to using +nightly. -orjson is tested for amd64, aarch64, and arm7 on Linux. It is tested for -amd64 on macOS and cross-compiles for aarch64. For Windows it is tested on -amd64. +orjson is tested on native hardware for amd64, aarch64, and i686 on Linux. It is +cross-compiled and may be tested via emulation for arm7, ppc64le, and s390x. It +is tested for aarch64 on macOS and cross-compiles for amd64. For +Windows it is tested on amd64, i686, and aarch64. There are no runtime dependencies other than libc. -orjson's tests are included in the source distribution on PyPI. The -requirements to run the tests are specified in `test/requirements.txt`. The -tests should be run as part of the build. It can be run with -`pytest -q test`. +The source distribution on PyPI contains all dependencies' source and can be +built without network access. The file can be downloaded from +`https://files.pythonhosted.org/packages/source/o/orjson/orjson-${version}.tar.gz`. + +orjson's tests are included in the source distribution on PyPI. The tests +require only `pytest`. There are optional packages such as `pytz` and `numpy` +listed in `test/requirements.txt` and used in ~10% of tests. Not having these +dependencies causes the tests needing them to skip. Tests can be run +with `pytest -q test`. ## License -orjson was written by ijl <>, copyright 2018 - 2022, licensed -under both the Apache 2 and MIT licenses. +orjson was written by ijl <>, copyright 2018 - 2026, with +some source files available under the Mozilla Public License 2.0 and some +available under your choice of the Apache 2 license or MIT license. diff --git a/bench/benchmark_dumps.py b/bench/benchmark_dumps.py index 205d1d47..ee01f8da 100644 --- a/bench/benchmark_dumps.py +++ b/bench/benchmark_dumps.py @@ -1,19 +1,19 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2020-2026), Aarni Koskela (2021) from json import loads as json_loads import pytest -from .data import fixtures, libraries -from .util import read_fixture_obj +from .data import FIXTURE_AS_OBJECTS, FIXTURE_NAMES, LIBRARIES -@pytest.mark.parametrize("library", libraries) -@pytest.mark.parametrize("fixture", fixtures) +@pytest.mark.parametrize("library", LIBRARIES) +@pytest.mark.parametrize("fixture", FIXTURE_NAMES) def test_dumps(benchmark, fixture, library): - dumper, loader = libraries[library] + dumper, _ = LIBRARIES[library] benchmark.group = f"{fixture} serialization" benchmark.extra_info["lib"] = library - data = read_fixture_obj(f"{fixture}.xz") - benchmark.extra_info["correct"] = json_loads(dumper(data)) == data + data = FIXTURE_AS_OBJECTS[fixture] + benchmark.extra_info["correct"] = json_loads(dumper(data)) == data # type: ignore benchmark(dumper, data) diff --git a/bench/benchmark_empty.py b/bench/benchmark_empty.py deleted file mode 100644 index 6fb9425f..00000000 --- a/bench/benchmark_empty.py +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from json import loads as json_loads - -import pytest - -from .data import libraries - - -@pytest.mark.parametrize("data", ["[]", "{}", '""']) -@pytest.mark.parametrize("library", libraries) -def test_empty(benchmark, data, library): - dumper, loader = libraries[library] - benchmark.extra_info["correct"] = json_loads(dumper(loader(data))) == json_loads( - data - ) - benchmark(loader, data) diff --git a/bench/benchmark_loads.py b/bench/benchmark_loads.py index 75d7b5ed..3d7a99e1 100644 --- a/bench/benchmark_loads.py +++ b/bench/benchmark_loads.py @@ -1,21 +1,20 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2020-2026), Aarni Koskela (2021) from json import loads as json_loads import pytest -from .data import fixtures, libraries -from .util import read_fixture +from .data import FIXTURE_AS_BYTES, FIXTURE_NAMES, LIBRARIES -@pytest.mark.parametrize("fixture", fixtures) -@pytest.mark.parametrize("library", libraries) +@pytest.mark.parametrize("fixture", FIXTURE_NAMES) +@pytest.mark.parametrize("library", LIBRARIES) def test_loads(benchmark, fixture, library): - dumper, loader = libraries[library] + dumper, loader = LIBRARIES[library] benchmark.group = f"{fixture} deserialization" benchmark.extra_info["lib"] = library - data = read_fixture(f"{fixture}.xz") - benchmark.extra_info["correct"] = json_loads(dumper(loader(data))) == json_loads( - data - ) + data = FIXTURE_AS_BYTES[fixture] + correct = json_loads(dumper(loader(data))) == json_loads(data) # type: ignore + benchmark.extra_info["correct"] = correct benchmark(loader, data) diff --git a/bench/data.py b/bench/data.py index c0dbc6ef..a670d9b5 100644 --- a/bench/data.py +++ b/bench/data.py @@ -1,54 +1,43 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2019-2026), Aarni Koskela (2021) +import gc from json import dumps as _json_dumps from json import loads as json_loads -from rapidjson import dumps as _rapidjson_dumps -from rapidjson import loads as rapidjson_loads -from simplejson import dumps as _simplejson_dumps -from simplejson import loads as simplejson_loads -from ujson import dumps as _ujson_dumps -from ujson import loads as ujson_loads - -from orjson import dumps as _orjson_dumps +from orjson import dumps as orjson_dumps from orjson import loads as orjson_loads -# dumps wrappers that return UTF-8 - - -def orjson_dumps(obj): - return _orjson_dumps(obj) - - -def ujson_dumps(obj): - return _ujson_dumps(obj).encode("utf-8") - - -def rapidjson_dumps(obj): - return _rapidjson_dumps(obj).encode("utf-8") +from .util import read_fixture def json_dumps(obj): return _json_dumps(obj).encode("utf-8") -def simplejson_dumps(obj): - return _simplejson_dumps(obj).encode("utf-8") - - -# Add new libraries here (pair of UTF-8 dumper, loader) -libraries = { +LIBRARIES = { "orjson": (orjson_dumps, orjson_loads), - "ujson": (ujson_dumps, ujson_loads), "json": (json_dumps, json_loads), - "rapidjson": (rapidjson_dumps, rapidjson_loads), - "simplejson": (simplejson_dumps, simplejson_loads), } -# Add new JSON files here (corresponding to ../data/*.json.xz) -fixtures = [ + +FIXTURE_NAMES = ( "canada.json", "citm_catalog.json", "github.json", "twitter.json", -] +) + +FIXTURE_AS_BYTES = {name: read_fixture(f"{name}.xz") for name in FIXTURE_NAMES} + +FIXTURE_AS_OBJECTS = { + name: orjson_loads(FIXTURE_AS_BYTES[name]) for name in FIXTURE_NAMES +} + + +if hasattr(gc, "freeze"): + gc.freeze() +if hasattr(gc, "collect"): + gc.collect() +if hasattr(gc, "disable"): + gc.disable() diff --git a/bench/requirements.txt b/bench/requirements.txt index 7c8c2e4a..e6e35769 100644 --- a/bench/requirements.txt +++ b/bench/requirements.txt @@ -1,7 +1,6 @@ -memory-profiler +memory-profiler; python_version<"3.15" and implementation_name=="cpython" +pandas; python_version<"3.15" and implementation_name=="cpython" pytest-benchmark pytest-random-order -python-rapidjson -simplejson +seaborn; python_version<"3.15" and implementation_name=="cpython" tabulate -ujson diff --git a/bench/run_func b/bench/run_func index 88d06fc0..a84845ff 100755 --- a/bench/run_func +++ b/bench/run_func @@ -1,11 +1,14 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2018-2025), Aarni Koskela (2021) import sys import lzma import os +import gc -os.sched_setaffinity(os.getpid(), {0, 1}) +if hasattr(os, "sched_setaffinity"): + os.sched_setaffinity(os.getpid(), {0, 1}) from orjson import dumps, loads @@ -15,10 +18,17 @@ n = int(sys.argv[3]) if len(sys.argv) >= 4 else 1000 with lzma.open(filename, "r") as fileh: file_bytes = fileh.read() +if hasattr(gc, "freeze"): + gc.freeze() +if hasattr(gc, "collect"): + gc.collect() +if hasattr(gc, "disable"): + gc.disable() + if sys.argv[2] == "dumps": file_obj = loads(file_bytes) for _ in range(n): - dumps(file_obj) + _ = dumps(file_obj) elif sys.argv[2] == "loads": for _ in range(n): - loads(file_bytes) + _ = loads(file_bytes) diff --git a/bench/run_mem b/bench/run_mem deleted file mode 100755 index e62e755a..00000000 --- a/bench/run_mem +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import sys -import lzma -import gc - -import psutil - -filename = sys.argv[1] - -with lzma.open(filename, "r") as fileh: - fixture = fileh.read() - -proc = psutil.Process() - -lib_name = sys.argv[2] -if lib_name == "json": - from json import dumps, loads -elif lib_name == "orjson": - from orjson import dumps, loads -elif lib_name == "rapidjson": - from rapidjson import dumps, loads -elif lib_name == "simplejson": - from simplejson import dumps, loads -elif lib_name == "ujson": - from ujson import dumps, loads -else: - raise NotImplementedError - -gc.collect() - -mem_before = proc.memory_info().rss - -for _ in range(100): - val = loads(fixture) - -mem_after = proc.memory_info().rss - -mem_diff = mem_after - mem_before - -from json import loads as json_loads - -correct = 1 if (json_loads(fixture) == json_loads(dumps(loads(fixture)))) else 0 - -print(f"{mem_before},{mem_diff},{correct}") diff --git a/bench/util.py b/bench/util.py index 7771ab60..c8955bd4 100644 --- a/bench/util.py +++ b/bench/util.py @@ -1,12 +1,9 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2018-2022), Aarni Koskela (2021) import lzma import os -from functools import lru_cache from pathlib import Path -from typing import Any - -import orjson dirname = os.path.join(os.path.dirname(__file__), "../data") @@ -14,7 +11,6 @@ os.sched_setaffinity(os.getpid(), {0, 1}) -@lru_cache(maxsize=None) def read_fixture(filename: str) -> bytes: path = Path(dirname, filename) if path.suffix == ".xz": @@ -22,8 +18,3 @@ def read_fixture(filename: str) -> bytes: else: contents = path.read_bytes() return contents - - -@lru_cache(maxsize=None) -def read_fixture_obj(filename: str) -> Any: - return orjson.loads(read_fixture(filename)) diff --git a/build.rs b/build.rs index 731a5108..09b723ef 100644 --- a/build.rs +++ b/build.rs @@ -1,19 +1,73 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2021-2026) + +fn main() { + let python_config = pyo3_build_config::get(); + + if python_config.is_free_threaded() && std::env::var("ORJSON_BUILD_FREETHREADED").is_err() { + not_supported("free-threaded Python") + } + + #[allow(unused_variables)] + let is_64_bit_python = matches!(python_config.pointer_width, Some(64)); + + match python_config.implementation { + pyo3_build_config::PythonImplementation::CPython => { + println!("cargo:rustc-cfg=CPython"); + if python_config.abi3 { + println!("cargo:rustc-cfg=Py_LIMITED_ABI"); + } + #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] + if is_64_bit_python && !python_config.abi3 { + println!("cargo:rustc-cfg=feature=\"inline_int\""); + #[cfg(target_endian = "little")] + println!("cargo:rustc-cfg=feature=\"inline_str\""); + } + } + _ => not_supported(&python_config.implementation.to_string()), + } + + for cfg in python_config.build_script_outputs() { + println!("{cfg}"); + } + + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed=include/yyjson/*"); + println!("cargo:rerun-if-env-changed=CC"); + println!("cargo:rerun-if-env-changed=CFLAGS"); + println!("cargo:rerun-if-env-changed=LDFLAGS"); + println!("cargo:rerun-if-env-changed=ORJSON_BUILD_FREETHREADED"); + println!("cargo:rerun-if-env-changed=RUSTFLAGS"); + println!("cargo:rustc-check-cfg=cfg(CPython)"); + println!("cargo:rustc-check-cfg=cfg(GraalPy)"); + println!("cargo:rustc-check-cfg=cfg(optimize)"); + println!("cargo:rustc-check-cfg=cfg(Py_3_10)"); + println!("cargo:rustc-check-cfg=cfg(Py_3_11)"); + println!("cargo:rustc-check-cfg=cfg(Py_3_12)"); + println!("cargo:rustc-check-cfg=cfg(Py_3_13)"); + println!("cargo:rustc-check-cfg=cfg(Py_3_14)"); + println!("cargo:rustc-check-cfg=cfg(Py_3_15)"); + println!("cargo:rustc-check-cfg=cfg(Py_GIL_DISABLED)"); + println!("cargo:rustc-check-cfg=cfg(Py_LIMITED_ABI)"); + println!("cargo:rustc-check-cfg=cfg(PyPy)"); + + #[cfg(all(target_arch = "x86_64", not(target_os = "macos")))] + if is_64_bit_python { + println!("cargo:rustc-cfg=feature=\"avx512\""); + } -#[allow(dead_code)] -#[cfg(feature = "yyjson")] -fn build_yyjson() { cc::Build::new() .file("include/yyjson/yyjson.c") .include("include/yyjson") - .define("YYJSON_DISABLE_WRITER", "1") .define("YYJSON_DISABLE_NON_STANDARD", "1") - .compile("yyjson"); + .define("YYJSON_DISABLE_UTF8_VALIDATION", "1") + .define("YYJSON_DISABLE_UTILS", "1") + .define("YYJSON_DISABLE_WRITER", "1") + .compile("yyjson") } -fn main() { - pyo3_build_config::use_pyo3_cfgs(); - - #[cfg(feature = "yyjson")] - build_yyjson(); +fn not_supported(flavor: &str) { + let version = env!("CARGO_PKG_VERSION"); + eprintln!("\n\n\norjson v{version} does not support {flavor}\n\n\n"); + std::process::exit(1); } diff --git a/ci/azure-debug.yml b/ci/azure-debug.yml deleted file mode 100644 index 81c7d12f..00000000 --- a/ci/azure-debug.yml +++ /dev/null @@ -1,32 +0,0 @@ -parameters: - - name: extra - type: string - default : '' - - name: interpreter - type: string - default : '' - - name: compatibility - type: string - default : '' - - name: path - type: string - default : '' - - name: toolchain - type: string - default : '' - -steps: -- bash: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain $(toolchain) --profile minimal -y - displayName: rustup -- bash: PATH=$(path) rustup default $(toolchain) - displayName: ensure toolchain -- bash: PATH=$(path) $(interpreter) -m pip install --user --upgrade pip "maturin>=0.12.19,<0.13" wheel - displayName: build dependencies -- bash: PATH=$(path) $(interpreter) -m pip install --user -r test/requirements.txt -r integration/requirements.txt - displayName: test dependencies -- bash: PATH=$(path) maturin build --no-sdist --strip $(extra) --compatibility $(compatibility) --interpreter $(interpreter) - displayName: build debug -- bash: PATH=$(path) $(interpreter) -m pip install --user target/wheels/orjson*.whl - displayName: install -- bash: PATH=$(path) pytest -s -rxX -v test - displayName: pytest diff --git a/ci/azure-macos.yml b/ci/azure-macos.yml deleted file mode 100644 index 79d58513..00000000 --- a/ci/azure-macos.yml +++ /dev/null @@ -1,54 +0,0 @@ -parameters: - interpreter: '' - toolchain: '' - -steps: -- bash: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain $(toolchain) --profile minimal -y - displayName: rustup -- bash: rustup default $(toolchain) - displayName: ensure toolchain -- bash: pip install --upgrade pip "maturin>=0.12.19,<0.13" wheel - displayName: build dependencies -- bash: pip install -r test/requirements.txt -r integration/requirements.txt - displayName: test dependencies -- bash: PATH=$HOME/.cargo/bin:$PATH cargo fetch -- bash: PATH=$HOME/.cargo/bin:$PATH maturin build --no-sdist --release --strip --cargo-extra-args="--features=unstable-simd,yyjson" --interpreter $(interpreter) - env: - CC: "clang" - CFLAGS: "-O2 -fno-plt -flto=thin" - LDFLAGS: "-O2 -flto=thin -fuse-ld=lld -Wl,--as-needed" - displayName: build -- bash: pip install target/wheels/orjson*.whl - displayName: install -- bash: pytest -s -rxX -v test - displayName: pytest -- bash: pip uninstall -y numpy - displayName: remove optional packages -- bash: pytest -s -rxX -v test - displayName: pytest without optional packages -- bash: ./integration/run thread - displayName: thread -- bash: ./integration/run http - displayName: http -- bash: rustup target add aarch64-apple-darwin - displayName: rustup target -- bash: PATH=$HOME/.cargo/bin:$PATH PYO3_CROSS_LIB_DIR=$(python -c "import sysconfig;print(sysconfig.get_config_var('LIBDIR'))") maturin build --no-sdist --release --strip --cargo-extra-args="--features=unstable-simd" --interpreter $(interpreter) --universal2 - env: - CC: "clang" - CFLAGS: "-O2 -fno-plt -flto=thin" - LDFLAGS: "-O2 -flto=thin -fuse-ld=lld -Wl,--as-needed" - displayName: build universal2 -- bash: pip install --force-reinstall target/wheels/orjson*universal2.whl - displayName: install universal2 -- bash: pytest -s -rxX -v test - displayName: pytest universal2 -- bash: ./ci/deploy target/wheels/*_x86_64.whl - displayName: deploy x86_64 - env: - MATURIN_PASSWORD: $(TWINE_PASSWORD) - MATURIN_USERNAME: $(TWINE_USERNAME) -- bash: ./ci/deploy target/wheels/*_universal2.whl - displayName: deploy universal2 - env: - MATURIN_PASSWORD: $(TWINE_PASSWORD) - MATURIN_USERNAME: $(TWINE_USERNAME) diff --git a/ci/azure-pipelines.yml b/ci/azure-pipelines.yml deleted file mode 100644 index b092747f..00000000 --- a/ci/azure-pipelines.yml +++ /dev/null @@ -1,141 +0,0 @@ -variables: - toolchain: nightly-2022-06-22 - -jobs: - -- job: linux_sdist_stable - pool: - vmImage: ubuntu-22.04 - container: quay.io/pypa/manylinux_2_28_x86_64:latest - variables: - interpreter: python3.10 - path: /home/vsts_azpcontainer/.local/bin:/home/vsts_azpcontainer/.cargo/bin:/opt/python/cp310-cp310/bin:/opt/rh/gcc-toolset-11/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - target: x86_64-unknown-linux-gnu - toolchain: 1.54.0 - steps: - - checkout: self - - template: ./azure-sdist.yml - -- job: linux_debug - pool: - vmImage: ubuntu-22.04 - container: quay.io/pypa/manylinux_2_28_x86_64:latest - variables: - interpreter: python3.10 - compatibility: off - path: /home/vsts_azpcontainer/.local/bin:/home/vsts_azpcontainer/.cargo/bin:/opt/python/cp310-cp310/bin:/opt/rh/gcc-toolset-11/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - steps: - - checkout: self - - template: ./azure-debug.yml - -- job: macos_python310_amd64 - pool: - vmImage: macOS-10.15 - variables: - interpreter: python3.10 - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.10' - addToPath: true - - checkout: self - - template: ./azure-macos.yml - -- job: macos_python39_amd64 - pool: - vmImage: macOS-10.15 - variables: - interpreter: python3.9 - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.9' - addToPath: true - - checkout: self - - template: ./azure-macos.yml - -- job: macos_python38_amd64 - pool: - vmImage: macOS-10.15 - variables: - interpreter: python3.8 - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.8' - addToPath: true - - checkout: self - - template: ./azure-macos.yml - -- job: macos_python37_amd64 - pool: - vmImage: macOS-10.15 - variables: - interpreter: python3.7 - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - addToPath: true - - checkout: self - - template: ./azure-macos.yml - -- job: win_python310_amd64 - pool: - vmImage: windows-2022 - variables: - interpreter: C:\hostedtoolcache\windows\Python\3.10.5\x64\python.exe - target: x86_64-pc-windows-msvc - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.10' - addToPath: true - architecture: 'x64' - - checkout: self - - template: ./azure-win.yml - -- job: win_python39_amd64 - pool: - vmImage: windows-2022 - variables: - interpreter: C:\hostedtoolcache\windows\Python\3.9.13\x64\python.exe - target: x86_64-pc-windows-msvc - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.9' - addToPath: true - architecture: 'x64' - - checkout: self - - template: ./azure-win.yml - -- job: win_python38_amd64 - pool: - vmImage: windows-2022 - variables: - interpreter: C:\hostedtoolcache\windows\Python\3.8.10\x64\python.exe - target: x86_64-pc-windows-msvc - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.8' - addToPath: true - architecture: 'x64' - - checkout: self - - template: ./azure-win.yml - -- job: win_python37_amd64 - pool: - vmImage: windows-2022 - variables: - interpreter: C:\hostedtoolcache\windows\Python\3.7.9\x64\python.exe - target: x86_64-pc-windows-msvc - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - addToPath: true - architecture: 'x64' - - checkout: self - - template: ./azure-win.yml diff --git a/ci/azure-sdist.yml b/ci/azure-sdist.yml deleted file mode 100644 index 35eb32db..00000000 --- a/ci/azure-sdist.yml +++ /dev/null @@ -1,37 +0,0 @@ -parameters: - - name: interpreter - type: string - default : '' - - name: path - type: string - default : '' - - name: toolchain - type: string - default : '' - -steps: -- bash: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain $(toolchain) --profile minimal -y - displayName: rustup -- bash: PATH=$(path) rustup default $(toolchain) - displayName: ensure toolchain -- bash: PATH=$(path) $(interpreter) -m pip install --user --upgrade pip "maturin>=0.12.19,<0.13" wheel - displayName: build dependencies -- bash: PATH=$(path) $(interpreter) -m pip install --user -r test/requirements.txt -r integration/requirements.txt mypy==0.960 - displayName: test dependencies -- bash: PATH=$(path) maturin sdist - displayName: package sdist -- bash: PATH=$(path) $(interpreter) -m pip install --user target/wheels/orjson*.tar.gz - displayName: install -- bash: PATH=$(path) pytest -v test - displayName: pytest -- bash: PATH=$(path) ./integration/run thread - displayName: thread -- bash: PATH=$(path) ./integration/run http - displayName: http -- bash: PATH=$(path) ./integration/run typestubs - displayName: typestubs -- bash: PATH=$(path) ./ci/deploy target/wheels/*.tar.gz - displayName: deploy - env: - MATURIN_PASSWORD: $(TWINE_PASSWORD) - MATURIN_USERNAME: $(TWINE_USERNAME) diff --git a/ci/azure-win.yml b/ci/azure-win.yml deleted file mode 100644 index cfe1dbe0..00000000 --- a/ci/azure-win.yml +++ /dev/null @@ -1,34 +0,0 @@ -parameters: - interpreter: '' - target: '' - toolchain: '' - -steps: -- script: | - curl https://win.rustup.rs/x86_64 -o rustup-init.exe - rustup-init.exe -y --default-host $(target) --default-toolchain $(toolchain) --profile minimal - set PATH=%PATH%;%USERPROFILE%\.cargo\bin - rustup default $(toolchain) - echo "##vso[task.setvariable variable=PATH;]%PATH%;%USERPROFILE%\.cargo\bin" - displayName: rustup -- script: python.exe -m pip install --upgrade pip "maturin>=0.12.19,<0.13" wheel - displayName: build dependencies -- script: python.exe -m pip install -r test\requirements.txt -r integration\requirements.txt - displayName: test dependencies -- script: maturin.exe build --no-sdist --release --strip --interpreter $(interpreter) - displayName: build -- script: python.exe -m pip install orjson --no-index --find-links=D:\a\1\s\target\wheels - displayName: install -- script: python.exe -m pytest -s -rxX -v test - displayName: pytest -- script: python.exe -m pip uninstall -y numpy - displayName: remove optional packages -- script: python.exe -m pytest -s -rxX -v test - displayName: pytest without optional packages -- script: python.exe integration\thread - displayName: thread -- bash: ./ci/deploy /d/a/1/s/target/wheels/*.whl - displayName: deploy - env: - MATURIN_PASSWORD: $(TWINE_PASSWORD) - MATURIN_USERNAME: $(TWINE_USERNAME) diff --git a/ci/config.toml b/ci/config.toml new file mode 100644 index 00000000..d255e8fa --- /dev/null +++ b/ci/config.toml @@ -0,0 +1,11 @@ +[unstable] +build-std = ["core", "std", "alloc", "proc_macro", "panic_abort"] +trim-paths = true + +[target.x86_64-apple-darwin] +linker = "clang" +rustflags = ["-C", "target-cpu=x86-64-v2", "-Z", "tune-cpu=generic"] + +[target.aarch64-apple-darwin] +linker = "clang" +rustflags = ["-C", "target-cpu=apple-m1", "-Z", "tune-cpu=generic"] diff --git a/ci/deploy b/ci/deploy deleted file mode 100755 index 4ea57b00..00000000 --- a/ci/deploy +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -if [ -z ${DRONE_TAG+x} ]; then - tag=$(git name-rev --tags --name-only $(git rev-parse HEAD)) -else - tag="$DRONE_TAG" -fi - -echo "$tag" - -if [[ "$tag" == "undefined" ]]; then - echo "not on a tag" - exit 0 -fi - -maturin upload --skip-existing --username "$MATURIN_USERNAME" "$1" diff --git a/ci/drone.yml b/ci/drone.yml deleted file mode 100644 index 64a197c4..00000000 --- a/ci/drone.yml +++ /dev/null @@ -1,121 +0,0 @@ -# the repetition should be a function using jsonnet, but drone still -# expects YAML even with .jsonnet extension, yielding a parse error -kind: pipeline -name: linux_python310_aarch64 -platform: - arch: arm64 -steps: -- name: test - image: quay.io/pypa/manylinux_2_28_aarch64:latest - environment: - PATH: "/root/.local/bin:/root/.cargo/bin:/opt/python/cp310-cp310/bin:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" - MATURIN_USERNAME: - from_secret: twine_username - MATURIN_PASSWORD: - from_secret: twine_password - CC: "clang" - CFLAGS: "-O2 -fno-plt -flto=thin" - LDFLAGS: "-O2 -flto=thin -fuse-ld=lld -Wl,--as-needed" - RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=lld" - commands: - - yum update -y && yum install -y clang lld - - curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2022-06-22 --profile minimal -y - - python3.10 -m pip install --user --upgrade pip "maturin>=0.12.19,<0.13" wheel - - cargo fetch - - maturin build --no-sdist --release --strip --compatibility manylinux_2_28 --cargo-extra-args="--features=unstable-simd,yyjson" --interpreter python3.10 - - python3.10 -m pip install --user target/wheels/orjson*.whl - - python3.10 -m pip install --user -r test/requirements.txt -r integration/requirements.txt - - pytest -s -rxX -v test - - ./integration/run thread - - ./integration/run http - - ./ci/deploy target/wheels/*.whl ---- -kind: pipeline -name: linux_python39_aarch64 -platform: - arch: arm64 -steps: -- name: test - image: quay.io/pypa/manylinux_2_28_aarch64:latest - environment: - PATH: "/root/.local/bin:/root/.cargo/bin:/opt/python/cp39-cp39/bin:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" - MATURIN_USERNAME: - from_secret: twine_username - MATURIN_PASSWORD: - from_secret: twine_password - CC: "clang" - CFLAGS: "-O2 -fno-plt -flto=thin" - LDFLAGS: "-O2 -flto=thin -fuse-ld=lld -Wl,--as-needed" - RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=lld" - commands: - - yum update -y && yum install -y clang lld - - curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2022-06-22 --profile minimal -y - - python3.9 -m pip install --user --upgrade pip "maturin>=0.12.19,<0.13" wheel - - cargo fetch - - maturin build --no-sdist --release --strip --compatibility manylinux_2_28 --cargo-extra-args="--features=unstable-simd,yyjson" --interpreter python3.9 - - python3.9 -m pip install --user target/wheels/orjson*.whl - - python3.9 -m pip install --user -r test/requirements.txt -r integration/requirements.txt - - pytest -s -rxX -v test - - ./integration/run thread - - ./integration/run http - - ./ci/deploy target/wheels/*.whl ---- -kind: pipeline -name: linux_python38_aarch64 -platform: - arch: arm64 -steps: -- name: test - image: quay.io/pypa/manylinux_2_28_aarch64:latest - environment: - PATH: "/root/.local/bin:/root/.cargo/bin:/opt/python/cp38-cp38/bin:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" - MATURIN_USERNAME: - from_secret: twine_username - MATURIN_PASSWORD: - from_secret: twine_password - CC: "clang" - CFLAGS: "-O2 -fno-plt -flto=thin" - LDFLAGS: "-O2 -flto=thin -fuse-ld=lld -Wl,--as-needed" - RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=lld" - commands: - - yum update -y && yum install -y clang lld - - curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2022-06-22 --profile minimal -y - - python3.8 -m pip install --user --upgrade pip "maturin>=0.12.19,<0.13" wheel - - cargo fetch - - maturin build --no-sdist --release --strip --compatibility manylinux_2_28 --cargo-extra-args="--features=unstable-simd,yyjson" --interpreter python3.8 - - python3.8 -m pip install --user target/wheels/orjson*.whl - - python3.8 -m pip install --user -r test/requirements.txt -r integration/requirements.txt - - pytest -s -rxX -v test - - ./integration/run thread - - ./integration/run http - - ./ci/deploy target/wheels/*.whl ---- -kind: pipeline -name: linux_python37_aarch64 -platform: - arch: arm64 -steps: -- name: test - image: quay.io/pypa/manylinux_2_28_aarch64:latest - environment: - PATH: "/root/.local/bin:/root/.cargo/bin:/opt/python/cp37-cp37m/bin:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" - MATURIN_USERNAME: - from_secret: twine_username - MATURIN_PASSWORD: - from_secret: twine_password - CC: "clang" - CFLAGS: "-O2 -fno-plt -flto=thin" - LDFLAGS: "-O2 -flto=thin -fuse-ld=lld -Wl,--as-needed" - RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=lld" - commands: - - yum update -y && yum install -y clang lld - - curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2022-06-22 --profile minimal -y - - python3.7 -m pip install --user --upgrade pip "maturin>=0.12.19,<0.13" wheel - - cargo fetch - - maturin build --no-sdist --release --strip --compatibility manylinux_2_28 --cargo-extra-args="--features=unstable-simd,yyjson" --interpreter python3.7 - - python3.7 -m pip install --user target/wheels/orjson*.whl - - python3.7 -m pip install --user -r test/requirements.txt -r integration/requirements.txt - - pytest -s -rxX -v test - - ./integration/run thread - - ./integration/run http - - ./ci/deploy target/wheels/*.whl diff --git a/ci/sdist.toml b/ci/sdist.toml new file mode 100644 index 00000000..0e35b81f --- /dev/null +++ b/ci/sdist.toml @@ -0,0 +1,5 @@ +[source.crates-io] +replace-with = "vendored-sources" + +[source.vendored-sources] +directory = "include/cargo" diff --git a/data/issue331_1.json.xz b/data/issue331_1.json.xz new file mode 100644 index 00000000..c61ee5fe Binary files /dev/null and b/data/issue331_1.json.xz differ diff --git a/data/issue331_2.json.xz b/data/issue331_2.json.xz new file mode 100644 index 00000000..ad742f03 Binary files /dev/null and b/data/issue331_2.json.xz differ diff --git a/doc/deserialization.png b/doc/deserialization.png new file mode 100644 index 00000000..ad4104b5 Binary files /dev/null and b/doc/deserialization.png differ diff --git a/doc/serialization.png b/doc/serialization.png new file mode 100644 index 00000000..5a160234 Binary files /dev/null and b/doc/serialization.png differ diff --git a/include/yyjson/yyjson.c b/include/yyjson/yyjson.c index abaa716f..1031182d 100644 --- a/include/yyjson/yyjson.c +++ b/include/yyjson/yyjson.c @@ -1,39 +1,49 @@ /*============================================================================== - * Created by Yaoyuan on 2019/3/9. - * Copyright (C) 2019 Yaoyuan . - * - * Released under the MIT License: - * https://github.com/ibireme/yyjson/blob/master/LICENSE + Copyright (c) 2020 YaoYuan + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. *============================================================================*/ #include "yyjson.h" -#include #include /*============================================================================== - * Compile Hint Begin + * Warning Suppress *============================================================================*/ -/* warning suppress begin */ #if defined(__clang__) -# pragma clang diagnostic push # pragma clang diagnostic ignored "-Wunused-function" # pragma clang diagnostic ignored "-Wunused-parameter" # pragma clang diagnostic ignored "-Wunused-label" # pragma clang diagnostic ignored "-Wunused-macros" +# pragma clang diagnostic ignored "-Wunused-variable" #elif defined(__GNUC__) -# if (__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6) -# pragma GCC diagnostic push -# endif # pragma GCC diagnostic ignored "-Wunused-function" # pragma GCC diagnostic ignored "-Wunused-parameter" # pragma GCC diagnostic ignored "-Wunused-label" # pragma GCC diagnostic ignored "-Wunused-macros" +# pragma GCC diagnostic ignored "-Wunused-variable" #elif defined(_MSC_VER) -# pragma warning(push) # pragma warning(disable:4100) /* unreferenced formal parameter */ +# pragma warning(disable:4101) /* unreferenced variable */ # pragma warning(disable:4102) /* unreferenced label */ # pragma warning(disable:4127) /* conditional expression is constant */ # pragma warning(disable:4706) /* assignment within conditional expression */ @@ -45,7 +55,7 @@ * Version *============================================================================*/ -yyjson_api uint32_t yyjson_version(void) { +uint32_t yyjson_version(void) { return YYJSON_VERSION_HEX; } @@ -55,32 +65,6 @@ yyjson_api uint32_t yyjson_version(void) { * Flags *============================================================================*/ -/* gcc version check */ -#if defined(__GNUC__) -# if defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) -# define yyjson_gcc_available(major, minor, patch) \ - ((__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) \ - >= (major * 10000 + minor * 100 + patch)) -# elif defined(__GNUC_MINOR__) -# define yyjson_gcc_available(major, minor, patch) \ - ((__GNUC__ * 10000 + __GNUC_MINOR__ * 100) \ - >= (major * 10000 + minor * 100 + patch)) -# else -# define yyjson_gcc_available(major, minor, patch) \ - ((__GNUC__ * 10000) >= (major * 10000 + minor * 100 + patch)) -# endif -#else -# define yyjson_gcc_available(major, minor, patch) 0 -#endif - -/* real gcc check */ -#if !defined(__clang__) && !defined(__INTEL_COMPILER) && !defined(__ICC) && \ - defined(__GNUC__) && defined(__GNUC_MINOR__) -# define YYJSON_IS_REAL_GCC 1 -#else -# define YYJSON_IS_REAL_GCC 0 -#endif - /* msvc intrinsic */ #if YYJSON_MSC_VER >= 1400 # include @@ -191,83 +175,47 @@ yyjson_api uint32_t yyjson_version(void) { # else # define YYJSON_DOUBLE_MATH_CORRECT 0 # endif -#elif defined(__x86_64) || defined(__x86_64__) || \ - defined(__amd64) || defined(__amd64__) || \ - defined(_M_AMD64) || defined(_M_X64) || \ - defined(__ia64) || defined(_IA64) || defined(__IA64__) || \ - defined(__ia64__) || defined(_M_IA64) || defined(__itanium__) || \ - defined(__arm64) || defined(__arm64__) || \ - defined(__aarch64__) || defined(_M_ARM64) || \ - defined(__arm) || defined(__arm__) || defined(_ARM_) || \ - defined(_ARM) || defined(_M_ARM) || defined(__TARGET_ARCH_ARM) || \ - defined(mips) || defined(__mips) || defined(__mips__) || \ - defined(MIPS) || defined(_MIPS_) || defined(__MIPS__) || \ - defined(_ARCH_PPC64) || defined(__PPC64__) || \ - defined(__ppc64__) || defined(__powerpc64__) || \ - defined(__powerpc) || defined(__powerpc__) || defined(__POWERPC__) || \ - defined(__ppc__) || defined(__ppc) || defined(__PPC__) || \ - defined(__sparcv9) || defined(__sparc_v9__) || \ - defined(__sparc) || defined(__sparc__) || defined(__sparc64__) || \ - defined(__alpha) || defined(__alpha__) || defined(_M_ALPHA) || \ - defined(__or1k__) || defined(__OR1K__) || defined(OR1K) || \ - defined(__hppa) || defined(__hppa__) || defined(__HPPA__) || \ - defined(__riscv) || defined(__riscv__) || \ - defined(__s390__) || defined(__avr32__) || defined(__SH4__) || \ - defined(__e2k__) || defined(__arc__) || defined(__ARC64__) || \ - defined(__loongarch__) || defined(__nios2__) || defined(__ghs) || \ - defined(__microblaze__) || defined(__XTENSA__) || \ - defined(__EMSCRIPTEN__) || defined(__wasm__) -# define YYJSON_DOUBLE_MATH_CORRECT 1 +#elif defined(__mc68000__) || defined(__pnacl__) || defined(__native_client__) +# define YYJSON_DOUBLE_MATH_CORRECT 0 #else -# define YYJSON_DOUBLE_MATH_CORRECT 0 /* unknown, disable fast path */ -#endif - -/* - Microsoft Visual C++ 6.0 doesn't support converting number from u64 to f64: - error C2520: conversion from unsigned __int64 to double not implemented. - */ -#ifndef YYJSON_U64_TO_F64_NO_IMPL -# if (0 < YYJSON_MSC_VER) && (YYJSON_MSC_VER <= 1200) -# define YYJSON_U64_TO_F64_NO_IMPL 1 -# else -# define YYJSON_U64_TO_F64_NO_IMPL 0 -# endif +# define YYJSON_DOUBLE_MATH_CORRECT 1 #endif /* endian */ #if yyjson_has_include() -# include +# include /* POSIX */ #endif - #if yyjson_has_include() -# include -#elif yyjson_has_include() -# include +# include /* Linux */ #elif yyjson_has_include() -# include +# include /* BSD, Android */ +#elif yyjson_has_include() +# include /* BSD, Darwin */ #endif #define YYJSON_BIG_ENDIAN 4321 #define YYJSON_LITTLE_ENDIAN 1234 -#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ -# if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if defined(BYTE_ORDER) && BYTE_ORDER +# if defined(BIG_ENDIAN) && (BYTE_ORDER == BIG_ENDIAN) # define YYJSON_ENDIAN YYJSON_BIG_ENDIAN -# elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +# elif defined(LITTLE_ENDIAN) && (BYTE_ORDER == LITTLE_ENDIAN) # define YYJSON_ENDIAN YYJSON_LITTLE_ENDIAN # endif #elif defined(__BYTE_ORDER) && __BYTE_ORDER -# if __BYTE_ORDER == __BIG_ENDIAN +# if defined(__BIG_ENDIAN) && (__BYTE_ORDER == __BIG_ENDIAN) # define YYJSON_ENDIAN YYJSON_BIG_ENDIAN -# elif __BYTE_ORDER == __LITTLE_ENDIAN +# elif defined(__LITTLE_ENDIAN) && (__BYTE_ORDER == __LITTLE_ENDIAN) # define YYJSON_ENDIAN YYJSON_LITTLE_ENDIAN # endif -#elif defined(BYTE_ORDER) && BYTE_ORDER -# if BYTE_ORDER == BIG_ENDIAN +#elif defined(__BYTE_ORDER__) && __BYTE_ORDER__ +# if defined(__ORDER_BIG_ENDIAN__) && \ + (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) # define YYJSON_ENDIAN YYJSON_BIG_ENDIAN -# elif BYTE_ORDER == LITTLE_ENDIAN +# elif defined(__ORDER_LITTLE_ENDIAN__) && \ + (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) # define YYJSON_ENDIAN YYJSON_LITTLE_ENDIAN # endif @@ -278,21 +226,16 @@ yyjson_api uint32_t yyjson_version(void) { defined(__x86_64) || defined(__x86_64__) || \ defined(__amd64) || defined(__amd64__) || \ defined(_M_AMD64) || defined(_M_X64) || \ - defined(__ia64) || defined(_IA64) || defined(__IA64__) || \ - defined(__ia64__) || defined(_M_IA64) || defined(__itanium__) || \ + defined(_M_ARM) || defined(_M_ARM64) || \ defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ - defined(__alpha) || defined(__alpha__) || defined(_M_ALPHA) || \ - defined(__riscv) || defined(__riscv__) || \ defined(_MIPSEL) || defined(__MIPSEL) || defined(__MIPSEL__) || \ - defined(__EMSCRIPTEN__) || defined(__wasm__) + defined(__EMSCRIPTEN__) || defined(__wasm__) || \ + defined(__loongarch__) # define YYJSON_ENDIAN YYJSON_LITTLE_ENDIAN #elif (defined(__BIG_ENDIAN__) && __BIG_ENDIAN__ == 1) || \ defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ defined(_MIPSEB) || defined(__MIPSEB) || defined(__MIPSEB__) || \ - defined(_ARCH_PPC) || defined(_ARCH_PPC64) || \ - defined(__ppc) || defined(__ppc__) || \ - defined(__sparc) || defined(__sparc__) || defined(__sparc64__) || \ defined(__or1k__) || defined(__OR1K__) # define YYJSON_ENDIAN YYJSON_BIG_ENDIAN @@ -301,61 +244,40 @@ yyjson_api uint32_t yyjson_version(void) { #endif /* - Unaligned memory access detection. + This macro controls how yyjson handles unaligned memory accesses. - Some architectures cannot perform unaligned memory accesse, or unaligned memory - accesses can have a large performance penalty. Modern compilers can make some - optimizations for unaligned access. For example: https://godbolt.org/z/Ejo3Pa + By default, yyjson uses `memcpy()` for memory copying. This takes advantage of + the compiler's automatic optimizations to generate unaligned memory access + instructions when the target architecture supports it. - typedef struct { char c[2] } vec2; - void copy_vec2(vec2 *dst, vec2 *src) { - *dst = *src; - } - - Compiler may generate `load/store` or `move` instruction if target architecture - supports unaligned access, otherwise it may generate `call memcpy` instruction. + However, for some older compilers or architectures where `memcpy()` isn't + optimized well and may generate unnecessary function calls, consider defining + this macro as 1. In such cases, yyjson switches to manual byte-by-byte access, + potentially improving performance. An example of the generated assembly code on + the ARM platform can be found here: https://godbolt.org/z/334jjhxPT - We want to avoid `memcpy` calls, so we should disable unaligned access by - define `YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS` as 1 on these architectures. + As this flag has already been enabled for some common architectures in the + following code, users typically don't need to manually specify it. If users are + unsure about it, please review the generated assembly code or perform actual + benchmark to make an informed decision. */ #ifndef YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS -# if defined(i386) || defined(__i386) || defined(__i386__) || \ - defined(__i486__) || defined(__i586__) || defined(__i686__) || \ - defined(_X86_) || defined(__X86__) || defined(_M_IX86) || \ - defined(__I86__) || defined(__IA32__) || \ - defined(__THW_INTEL) || defined(__THW_INTEL__) || \ - defined(__x86_64) || defined(__x86_64__) || \ - defined(__amd64) || defined(__amd64__) || \ - defined(_M_AMD64) || defined(_M_X64) -# define YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS 0 /* x86 */ - -# elif defined(__ia64) || defined(_IA64) || defined(__IA64__) || \ +# if defined(__ia64) || defined(_IA64) || defined(__IA64__) || \ defined(__ia64__) || defined(_M_IA64) || defined(__itanium__) # define YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS 1 /* Itanium */ - -# elif defined(__arm64) || defined(__arm64__) || \ - defined(__AARCH64EL__) || defined(__AARCH64EB__) || \ - defined(__aarch64__) || defined(_M_ARM64) -# define YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS 0 /* ARM64 */ - -# elif defined(__ARM_ARCH_4__) || defined(__ARM_ARCH_4T__) || \ - defined(__ARM_ARCH_5TEJ__) || defined(__ARM_ARCH_5TE__) || \ - defined(__ARM_ARCH_6T2__) || defined(__ARM_ARCH_6KZ__) || \ - defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6K__) +# elif (defined(__arm__) || defined(__arm64__) || defined(__aarch64__)) && \ + (defined(__GNUC__) || defined(__clang__)) && \ + (!defined(__ARM_FEATURE_UNALIGNED) || !__ARM_FEATURE_UNALIGNED) # define YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS 1 /* ARM */ - -# elif defined(__ppc64__) || defined(__PPC64__) || \ - defined(__powerpc64__) || defined(_ARCH_PPC64) || \ - defined(__ppc) || defined(__ppc__) || defined(__PPC__) || \ - defined(__powerpc) || defined(__powerpc__) || defined(__POWERPC__) || \ - defined(_ARCH_PPC) || defined(_M_PPC) || \ - defined(__PPCGECKO__) || defined(__PPCBROADWAY__) || defined(_XENON) -# define YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS 0 /* PowerPC */ - +# elif defined(__sparc) || defined(__sparc__) +# define YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS 1 /* SPARC */ +# elif defined(__mips) || defined(__mips__) || defined(__MIPS__) +# define YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS 1 /* MIPS */ +# elif defined(__m68k__) || defined(M68000) +# define YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS 1 /* M68K */ # else -# define YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS 0 /* Unknown */ +# define YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS 0 # endif - #endif /* @@ -371,7 +293,7 @@ yyjson_api uint32_t yyjson_version(void) { JSON, the ratios below are used to determine the initial memory size. A too large ratio will waste memory, and a too small ratio will cause multiple - memory growths and degrade performance. Currently these ratios are generated + memory growths and degrade performance. Currently, these ratios are generated with some commonly used JSON datasets. */ #define YYJSON_READER_ESTIMATED_PRETTY_RATIO 16 @@ -379,6 +301,15 @@ yyjson_api uint32_t yyjson_version(void) { #define YYJSON_WRITER_ESTIMATED_PRETTY_RATIO 32 #define YYJSON_WRITER_ESTIMATED_MINIFY_RATIO 18 +/* The initial and maximum size of the memory pool's chunk in yyjson_mut_doc. */ +#define YYJSON_MUT_DOC_STR_POOL_INIT_SIZE 0x100 +#define YYJSON_MUT_DOC_STR_POOL_MAX_SIZE 0x10000000 +#define YYJSON_MUT_DOC_VAL_POOL_INIT_SIZE (0x10 * sizeof(yyjson_mut_val)) +#define YYJSON_MUT_DOC_VAL_POOL_MAX_SIZE (0x1000000 * sizeof(yyjson_mut_val)) + +/* The minimum size of the dynamic allocator's chunk. */ +#define YYJSON_ALC_DYN_MIN_SIZE 0x1000 + /* Default value for compile-time options. */ #ifndef YYJSON_DISABLE_READER #define YYJSON_DISABLE_READER 0 @@ -386,14 +317,21 @@ yyjson_api uint32_t yyjson_version(void) { #ifndef YYJSON_DISABLE_WRITER #define YYJSON_DISABLE_WRITER 0 #endif +#ifndef YYJSON_DISABLE_UTILS +#define YYJSON_DISABLE_UTILS 0 +#endif #ifndef YYJSON_DISABLE_FAST_FP_CONV #define YYJSON_DISABLE_FAST_FP_CONV 0 #endif #ifndef YYJSON_DISABLE_NON_STANDARD #define YYJSON_DISABLE_NON_STANDARD 0 #endif - - +#ifndef YYJSON_DISABLE_UTF8_VALIDATION +#define YYJSON_DISABLE_UTF8_VALIDATION 0 +#endif +#ifndef YYJSON_READER_CONTAINER_RECURSION_LIMIT +#define YYJSON_READER_CONTAINER_RECURSION_LIMIT 1024 +#endif /*============================================================================== * Macros @@ -406,14 +344,15 @@ yyjson_api uint32_t yyjson_version(void) { #define repeat8(x) { x x x x x x x x } #define repeat16(x) { x x x x x x x x x x x x x x x x } -#define repeat2_incr(x) { x(0) x(1) } -#define repeat4_incr(x) { x(0) x(1) x(2) x(3) } -#define repeat8_incr(x) { x(0) x(1) x(2) x(3) x(4) x(5) x(6) x(7) } -#define repeat16_incr(x) { x(0) x(1) x(2) x(3) x(4) x(5) x(6) x(7) \ - x(8) x(9) x(10) x(11) x(12) x(13) x(14) x(15) } -#define repeat_in_1_18(x) { x(1) x(2) x(3) x(4) x(5) x(6) x(7) \ - x(8) x(9) x(10) x(11) x(12) x(13) x(14) x(15) \ - x(16) x(17) x(18) } +#define repeat2_incr(x) { x(0) x(1) } +#define repeat4_incr(x) { x(0) x(1) x(2) x(3) } +#define repeat8_incr(x) { x(0) x(1) x(2) x(3) x(4) x(5) x(6) x(7) } +#define repeat16_incr(x) { x(0) x(1) x(2) x(3) x(4) x(5) x(6) x(7) \ + x(8) x(9) x(10) x(11) x(12) x(13) x(14) x(15) } + +#define repeat_in_1_18(x) { x(1) x(2) x(3) x(4) x(5) x(6) x(7) x(8) \ + x(9) x(10) x(11) x(12) x(13) x(14) x(15) x(16) \ + x(17) x(18) } /* Macros used to provide branch prediction information for compiler. */ #undef likely @@ -437,6 +376,33 @@ yyjson_api uint32_t yyjson_version(void) { #undef U64 #define U64(hi, lo) ((((u64)hi##UL) << 32U) + lo##UL) +/* Used to cast away (remove) const qualifier. */ +#define constcast(type) (type)(void *)(size_t)(const void *) + +/* flag test */ +#define has_read_flag(_flag) false +#define has_write_flag(_flag) unlikely(write_flag_eq(flg, YYJSON_WRITE_##_flag)) + +static_inline bool read_flag_eq(yyjson_read_flag flg, yyjson_read_flag chk) { +#if YYJSON_DISABLE_NON_STANDARD + if (chk == YYJSON_READ_ALLOW_INF_AND_NAN || + chk == YYJSON_READ_ALLOW_COMMENTS || + chk == YYJSON_READ_ALLOW_TRAILING_COMMAS || + chk == YYJSON_READ_ALLOW_INVALID_UNICODE) + return false; /* this should be evaluated at compile-time */ +#endif + return (flg & chk) != 0; +} + +static_inline bool write_flag_eq(yyjson_write_flag flg, yyjson_write_flag chk) { +#if YYJSON_DISABLE_NON_STANDARD + if (chk == YYJSON_WRITE_ALLOW_INF_AND_NAN || + chk == YYJSON_WRITE_ALLOW_INVALID_UNICODE) + return false; /* this should be evaluated at compile-time */ +#endif + return (flg & chk) != 0; +} + /*============================================================================== @@ -451,9 +417,13 @@ yyjson_api uint32_t yyjson_version(void) { #undef USIZE_MAX #define USIZE_MAX ((usize)(~(usize)0)) -/* Maximum number of digits for reading u64 safety. */ +/* Maximum number of digits for reading u32/u64/usize safety (not overflow). */ +#undef U32_SAFE_DIG +#define U32_SAFE_DIG 9 /* u32 max is 4294967295, 10 digits */ #undef U64_SAFE_DIG -#define U64_SAFE_DIG 19 +#define U64_SAFE_DIG 19 /* u64 max is 18446744073709551615, 20 digits */ +#undef USIZE_SAFE_DIG +#define USIZE_SAFE_DIG (sizeof(usize) == 8 ? U64_SAFE_DIG : U32_SAFE_DIG) @@ -464,8 +434,12 @@ yyjson_api uint32_t yyjson_version(void) { /* Inf raw value (positive) */ #define F64_RAW_INF U64(0x7FF00000, 0x00000000) -/* NaN raw value (positive, without payload) */ +/* NaN raw value (quiet NaN, no payload, no sign) */ +#if defined(__hppa__) || (defined(__mips__) && !defined(__mips_nan2008)) +#define F64_RAW_NAN U64(0x7FF7FFFF, 0xFFFFFFFF) +#else #define F64_RAW_NAN U64(0x7FF80000, 0x00000000) +#endif /* double number bits */ #define F64_BITS 64 @@ -532,11 +506,11 @@ __extension__ typedef unsigned __int128 u128; #endif /** 16/32/64-bit vector */ -typedef struct v16 { char c1, c2; } v16; -typedef struct v32 { char c1, c2, c3, c4; } v32; -typedef struct v64 { char c1, c2, c3, c4, c5, c6, c7, c8; } v64; +typedef struct v16 { char c[2]; } v16; +typedef struct v32 { char c[4]; } v32; +typedef struct v64 { char c[8]; } v64; -/** 16/32/64-bit vector union, used for unaligned memory access on modern CPU */ +/** 16/32/64-bit vector union */ typedef union v16_uni { v16 v; u16 u; } v16_uni; typedef union v32_uni { v32 v; u32 u; } v32_uni; typedef union v64_uni { v64 v; u64 u; } v64_uni; @@ -547,121 +521,164 @@ typedef union v64_uni { v64 v; u64 u; } v64_uni; * Load/Store Utils *============================================================================*/ -#define byte_move_idx(x) ((u8 *)dst)[x] = ((u8 *)src)[x]; +#if YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS + +#define byte_move_idx(x) ((char *)dst)[x] = ((const char *)src)[x]; + +static_inline void byte_copy_2(void *dst, const void *src) { + repeat2_incr(byte_move_idx) +} + +static_inline void byte_copy_4(void *dst, const void *src) { + repeat4_incr(byte_move_idx) +} + +static_inline void byte_copy_8(void *dst, const void *src) { + repeat8_incr(byte_move_idx) +} + +static_inline void byte_copy_16(void *dst, const void *src) { + repeat16_incr(byte_move_idx) +} static_inline void byte_move_2(void *dst, const void *src) { -#if YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS - repeat2_incr(byte_move_idx); -#else - memmove(dst, src, 2); -#endif + repeat2_incr(byte_move_idx) } static_inline void byte_move_4(void *dst, const void *src) { -#if YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS - repeat4_incr(byte_move_idx); -#else - memmove(dst, src, 4); -#endif + repeat4_incr(byte_move_idx) } static_inline void byte_move_8(void *dst, const void *src) { -#if YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS - repeat8_incr(byte_move_idx); -#else - memmove(dst, src, 8); -#endif + repeat8_incr(byte_move_idx) } static_inline void byte_move_16(void *dst, const void *src) { -#if YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS - repeat16_incr(byte_move_idx); -#else - memmove(dst, src, 16); -#endif + repeat16_incr(byte_move_idx) } -static_inline void byte_copy_2(void *dst, const void *src) { -#if YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS - repeat2_incr(byte_move_idx); +static_inline bool byte_match_2(void *buf, const char *pat) { + return + ((char *)buf)[0] == ((const char *)pat)[0] && + ((char *)buf)[1] == ((const char *)pat)[1]; +} + +static_inline bool byte_match_4(void *buf, const char *pat) { + return + ((char *)buf)[0] == ((const char *)pat)[0] && + ((char *)buf)[1] == ((const char *)pat)[1] && + ((char *)buf)[2] == ((const char *)pat)[2] && + ((char *)buf)[3] == ((const char *)pat)[3]; +} + +static_inline u16 byte_load_2(const void *src) { + v16_uni uni; + uni.v.c[0] = ((const char *)src)[0]; + uni.v.c[1] = ((const char *)src)[1]; + return uni.u; +} + +static_inline u32 byte_load_3(const void *src) { + v32_uni uni; + uni.v.c[0] = ((const char *)src)[0]; + uni.v.c[1] = ((const char *)src)[1]; + uni.v.c[2] = ((const char *)src)[2]; + uni.v.c[3] = 0; + return uni.u; +} + +static_inline u32 byte_load_4(const void *src) { + v32_uni uni; + uni.v.c[0] = ((const char *)src)[0]; + uni.v.c[1] = ((const char *)src)[1]; + uni.v.c[2] = ((const char *)src)[2]; + uni.v.c[3] = ((const char *)src)[3]; + return uni.u; +} + +#undef byte_move_expr + #else + +static_inline void byte_copy_2(void *dst, const void *src) { memcpy(dst, src, 2); -#endif } static_inline void byte_copy_4(void *dst, const void *src) { -#if YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS - repeat4_incr(byte_move_idx); -#else memcpy(dst, src, 4); -#endif } static_inline void byte_copy_8(void *dst, const void *src) { -#if YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS - repeat8_incr(byte_move_idx); -#else memcpy(dst, src, 8); -#endif } static_inline void byte_copy_16(void *dst, const void *src) { -#if YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS - repeat16_incr(byte_move_idx); -#else memcpy(dst, src, 16); -#endif +} + +static_inline void byte_move_2(void *dst, const void *src) { + u16 tmp; + memcpy(&tmp, src, 2); + memcpy(dst, &tmp, 2); +} + +static_inline void byte_move_4(void *dst, const void *src) { + u32 tmp; + memcpy(&tmp, src, 4); + memcpy(dst, &tmp, 4); +} + +static_inline void byte_move_8(void *dst, const void *src) { + u64 tmp; + memcpy(&tmp, src, 8); + memcpy(dst, &tmp, 8); +} + +static_inline void byte_move_16(void *dst, const void *src) { + char *pdst = (char *)dst; + const char *psrc = (const char *)src; + u64 tmp1, tmp2; + memcpy(&tmp1, psrc, 8); + memcpy(&tmp2, psrc + 8, 8); + memcpy(pdst, &tmp1, 8); + memcpy(pdst + 8, &tmp2, 8); } static_inline bool byte_match_2(void *buf, const char *pat) { -#if YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS - return - ((u8 *)buf)[0] == ((const u8 *)pat)[0] && - ((u8 *)buf)[1] == ((const u8 *)pat)[1]; -#else v16_uni u1, u2; - u1.v = *(const v16 *)pat; - u2.v = *(const v16 *)buf; + memcpy(&u1, buf, 2); + memcpy(&u2, pat, 2); return u1.u == u2.u; -#endif } static_inline bool byte_match_4(void *buf, const char *pat) { -#if YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS - return - ((u8 *)buf)[0] == ((const u8 *)pat)[0] && - ((u8 *)buf)[1] == ((const u8 *)pat)[1] && - ((u8 *)buf)[2] == ((const u8 *)pat)[2] && - ((u8 *)buf)[3] == ((const u8 *)pat)[3]; -#else v32_uni u1, u2; - u1.v = *(const v32 *)pat; - u2.v = *(const v32 *)buf; + memcpy(&u1, buf, 4); + memcpy(&u2, pat, 4); return u1.u == u2.u; -#endif } static_inline u16 byte_load_2(const void *src) { v16_uni uni; - uni.v = *(v16 *)src; + memcpy(&uni, src, 2); return uni.u; } static_inline u32 byte_load_3(const void *src) { v32_uni uni; - ((v16_uni *)&uni)->v = *(v16 *)src; - uni.v.c3 = ((char *)src)[2]; - uni.v.c4 = 0; + memcpy(&uni, src, 2); + uni.v.c[2] = ((const char *)src)[2]; + uni.v.c[3] = 0; return uni.u; } static_inline u32 byte_load_4(const void *src) { v32_uni uni; - uni.v = *(v32 *)src; + memcpy(&uni, src, 4); return uni.u; } -#undef byte_move_expr +#endif @@ -670,37 +687,20 @@ static_inline u32 byte_load_4(const void *src) { * These functions are used to detect and convert NaN and Inf numbers. *============================================================================*/ -/** - This union is used to avoid violating the strict aliasing rule in C. - `memcpy` can be used in both C and C++, but it may reduce performance without - compiler optimization. - */ -typedef union { u64 u; f64 f; } f64_uni; - /** Convert raw binary to double. */ static_inline f64 f64_from_raw(u64 u) { -#ifndef __cplusplus - f64_uni uni; - uni.u = u; - return uni.f; -#else + /* use memcpy to avoid violating the strict aliasing rule */ f64 f; memcpy(&f, &u, 8); return f; -#endif } /** Convert double to raw binary. */ static_inline u64 f64_to_raw(f64 f) { -#ifndef __cplusplus - f64_uni uni; - uni.f = f; - return uni.u; -#else + /* use memcpy to avoid violating the strict aliasing rule */ u64 u; memcpy(&u, &f, 8); return u; -#endif } /** Get raw 'infinity' with sign. */ @@ -750,8 +750,7 @@ static_inline f64 normalized_u64_to_f64(u64 val) { /** Returns whether the size is overflow after increment. */ static_inline bool size_add_is_overflow(usize size, usize add) { - usize val = size + add; - return (val < size) | (val < add); + return size > (size + add); } /** Returns whether the size is power of 2 (size should not be 0). */ @@ -786,15 +785,6 @@ static_inline void *mem_align_up(void *mem, usize align) { return mem; } -/** Align address downwards. */ -static_inline void *mem_align_down(void *mem, usize align) { - usize size; - memcpy(&size, &mem, sizeof(usize)); - size = size_align_down(size, align); - memcpy(&mem, &size, sizeof(usize)); - return mem; -} - /*============================================================================== @@ -965,7 +955,7 @@ static void *default_malloc(void *ctx, usize size) { return malloc(size); } -static void *default_realloc(void *ctx, void *ptr, usize size) { +static void *default_realloc(void *ctx, void *ptr, usize old_size, usize size) { return realloc(ptr, size); } @@ -982,31 +972,67 @@ static const yyjson_alc YYJSON_DEFAULT_ALC = { +/*============================================================================== + * Null Memory Allocator + * + * This allocator is just a placeholder to ensure that the internal + * malloc/realloc/free function pointers are not null. + *============================================================================*/ + +static void *null_malloc(void *ctx, usize size) { + return NULL; +} + +static void *null_realloc(void *ctx, void *ptr, usize old_size, usize size) { + return NULL; +} + +static void null_free(void *ctx, void *ptr) { + return; +} + +static const yyjson_alc YYJSON_NULL_ALC = { + null_malloc, + null_realloc, + null_free, + NULL +}; + + + /*============================================================================== * Pool Memory Allocator - * This is a simple memory allocator that uses linked list memory chunk. - * The following code will be executed only when the library user creates - * this allocator manually. + * + * This allocator is initialized with a fixed-size buffer. + * The buffer is split into multiple memory chunks for memory allocation. *============================================================================*/ -/** chunk header */ +/** memory chunk header */ typedef struct pool_chunk { - usize size; /* chunk memory size (include chunk header) */ - struct pool_chunk *next; + usize size; /* chunk memory size, include chunk header */ + struct pool_chunk *next; /* linked list, nullable */ + /* char mem[]; flexible array member */ } pool_chunk; -/** ctx header */ +/** allocator ctx header */ typedef struct pool_ctx { - usize size; /* total memory size (include ctx header) */ - pool_chunk *free_list; + usize size; /* total memory size, include ctx header */ + pool_chunk *free_list; /* linked list, nullable */ + /* pool_chunk chunks[]; flexible array member */ } pool_ctx; +/** align up the input size to chunk size */ +static_inline void pool_size_align(usize *size) { + *size = size_align_up(*size, sizeof(pool_chunk)) + sizeof(pool_chunk); +} + static void *pool_malloc(void *ctx_ptr, usize size) { + /* assert(size != 0) */ pool_ctx *ctx = (pool_ctx *)ctx_ptr; pool_chunk *next, *prev = NULL, *cur = ctx->free_list; - if (unlikely(size == 0 || size >= ctx->size)) return NULL; - size = size_align_up(size, sizeof(pool_chunk)) + sizeof(pool_chunk); + if (unlikely(size >= ctx->size)) return NULL; + pool_size_align(&size); while (cur) { if (cur->size < size) { @@ -1033,6 +1059,7 @@ static void *pool_malloc(void *ctx_ptr, usize size) { } static void pool_free(void *ctx_ptr, void *ptr) { + /* assert(ptr != NULL) */ pool_ctx *ctx = (pool_ctx *)ctx_ptr; pool_chunk *cur = ((pool_chunk *)ptr) - 1; pool_chunk *prev = NULL, *next = ctx->free_list; @@ -1057,26 +1084,17 @@ static void pool_free(void *ctx_ptr, void *ptr) { } } -static void *pool_realloc(void *ctx_ptr, void *ptr, usize size) { +static void *pool_realloc(void *ctx_ptr, void *ptr, + usize old_size, usize size) { + /* assert(ptr != NULL && size != 0 && old_size < size) */ pool_ctx *ctx = (pool_ctx *)ctx_ptr; pool_chunk *cur = ((pool_chunk *)ptr) - 1, *prev, *next, *tmp; - usize free_size; - void *new_ptr; - if (unlikely(size == 0 || size >= ctx->size)) return NULL; - size = size_align_up(size, sizeof(pool_chunk)) + sizeof(pool_chunk); - - /* reduce size */ - if (unlikely(size <= cur->size)) { - free_size = cur->size - size; - if (free_size >= sizeof(pool_chunk) * 2) { - tmp = (pool_chunk *)(void *)((u8 *)cur + cur->size - free_size); - tmp->size = free_size; - pool_free(ctx_ptr, (void *)(tmp + 1)); - cur->size -= free_size; - } - return ptr; - } + /* check size */ + if (unlikely(size >= ctx->size)) return NULL; + pool_size_align(&old_size); + pool_size_align(&size); + if (unlikely(old_size == size)) return ptr; /* find next and prev chunk */ prev = NULL; @@ -1086,10 +1104,9 @@ static void *pool_realloc(void *ctx_ptr, void *ptr, usize size) { next = next->next; } - /* merge to higher chunk if they are contiguous */ - if ((u8 *)cur + cur->size == (u8 *)next && - cur->size + next->size >= size) { - free_size = cur->size + next->size - size; + if ((u8 *)cur + cur->size == (u8 *)next && cur->size + next->size >= size) { + /* merge to higher chunk if they are contiguous */ + usize free_size = cur->size + next->size - size; if (free_size > sizeof(pool_chunk) * 2) { tmp = (pool_chunk *)(void *)((u8 *)cur + size); if (prev) prev->next = tmp; @@ -1103,22 +1120,24 @@ static void *pool_realloc(void *ctx_ptr, void *ptr, usize size) { cur->size += next->size; } return ptr; + } else { + /* fallback to malloc and memcpy */ + void *new_ptr = pool_malloc(ctx_ptr, size - sizeof(pool_chunk)); + if (new_ptr) { + memcpy(new_ptr, ptr, cur->size - sizeof(pool_chunk)); + pool_free(ctx_ptr, ptr); + } + return new_ptr; } - - /* fallback to malloc and memcpy */ - new_ptr = pool_malloc(ctx_ptr, size - sizeof(pool_chunk)); - if (new_ptr) { - memcpy(new_ptr, ptr, cur->size - sizeof(pool_chunk)); - pool_free(ctx_ptr, ptr); - } - return new_ptr; } bool yyjson_alc_pool_init(yyjson_alc *alc, void *buf, usize size) { pool_chunk *chunk; pool_ctx *ctx; - if (unlikely(!alc || size < sizeof(pool_ctx) * 4)) return false; + if (unlikely(!alc)) return false; + *alc = YYJSON_NULL_ALC; + if (size < sizeof(pool_ctx) * 4) return false; ctx = (pool_ctx *)mem_align_up(buf, sizeof(pool_ctx)); if (unlikely(!ctx)) return false; size -= (usize)((u8 *)ctx - (u8 *)buf); @@ -1139,6 +1158,161 @@ bool yyjson_alc_pool_init(yyjson_alc *alc, void *buf, usize size) { +/*============================================================================== + * Dynamic Memory Allocator + * + * This allocator allocates memory on demand and does not immediately release + * unused memory. Instead, it places the unused memory into a freelist for + * potential reuse in the future. It is only when the entire allocator is + * destroyed that all previously allocated memory is released at once. + *============================================================================*/ + +/** memory chunk header */ +typedef struct dyn_chunk { + usize size; /* chunk size, include header */ + struct dyn_chunk *next; + /* char mem[]; flexible array member */ +} dyn_chunk; + +/** allocator ctx header */ +typedef struct { + dyn_chunk free_list; /* dummy header, sorted from small to large */ + dyn_chunk used_list; /* dummy header */ +} dyn_ctx; + +/** align up the input size to chunk size */ +static_inline bool dyn_size_align(usize *size) { + usize alc_size = *size + sizeof(dyn_chunk); + alc_size = size_align_up(alc_size, YYJSON_ALC_DYN_MIN_SIZE); + if (unlikely(alc_size < *size)) return false; /* overflow */ + *size = alc_size; + return true; +} + +/** remove a chunk from list (the chunk must already be in the list) */ +static_inline void dyn_chunk_list_remove(dyn_chunk *list, dyn_chunk *chunk) { + dyn_chunk *prev = list, *cur; + for (cur = prev->next; cur; cur = cur->next) { + if (cur == chunk) { + prev->next = cur->next; + cur->next = NULL; + return; + } + prev = cur; + } +} + +/** add a chunk to list header (the chunk must not be in the list) */ +static_inline void dyn_chunk_list_add(dyn_chunk *list, dyn_chunk *chunk) { + chunk->next = list->next; + list->next = chunk; +} + +static void *dyn_malloc(void *ctx_ptr, usize size) { + /* assert(size != 0) */ + const yyjson_alc def = YYJSON_DEFAULT_ALC; + dyn_ctx *ctx = (dyn_ctx *)ctx_ptr; + dyn_chunk *chunk, *prev, *next; + if (unlikely(!dyn_size_align(&size))) return NULL; + + /* freelist is empty, create new chunk */ + if (!ctx->free_list.next) { + chunk = (dyn_chunk *)def.malloc(def.ctx, size); + if (unlikely(!chunk)) return NULL; + chunk->size = size; + chunk->next = NULL; + dyn_chunk_list_add(&ctx->used_list, chunk); + return (void *)(chunk + 1); + } + + /* find a large enough chunk, or resize the largest chunk */ + prev = &ctx->free_list; + while (true) { + chunk = prev->next; + if (chunk->size >= size) { /* enough size, reuse this chunk */ + prev->next = chunk->next; + dyn_chunk_list_add(&ctx->used_list, chunk); + return (void *)(chunk + 1); + } + if (!chunk->next) { /* resize the largest chunk */ + chunk = (dyn_chunk *)def.realloc(def.ctx, chunk, chunk->size, size); + if (unlikely(!chunk)) return NULL; + prev->next = NULL; + chunk->size = size; + dyn_chunk_list_add(&ctx->used_list, chunk); + return (void *)(chunk + 1); + } + prev = chunk; + } +} + +static void *dyn_realloc(void *ctx_ptr, void *ptr, + usize old_size, usize size) { + /* assert(ptr != NULL && size != 0 && old_size < size) */ + const yyjson_alc def = YYJSON_DEFAULT_ALC; + dyn_ctx *ctx = (dyn_ctx *)ctx_ptr; + dyn_chunk *prev, *next, *new_chunk; + dyn_chunk *chunk = (dyn_chunk *)ptr - 1; + if (unlikely(!dyn_size_align(&size))) return NULL; + if (chunk->size >= size) return ptr; + + dyn_chunk_list_remove(&ctx->used_list, chunk); + new_chunk = (dyn_chunk *)def.realloc(def.ctx, chunk, chunk->size, size); + if (likely(new_chunk)) { + new_chunk->size = size; + chunk = new_chunk; + } + dyn_chunk_list_add(&ctx->used_list, chunk); + return new_chunk ? (void *)(new_chunk + 1) : NULL; +} + +static void dyn_free(void *ctx_ptr, void *ptr) { + /* assert(ptr != NULL) */ + dyn_ctx *ctx = (dyn_ctx *)ctx_ptr; + dyn_chunk *chunk = (dyn_chunk *)ptr - 1, *prev; + + dyn_chunk_list_remove(&ctx->used_list, chunk); + for (prev = &ctx->free_list; prev; prev = prev->next) { + if (!prev->next || prev->next->size >= chunk->size) { + chunk->next = prev->next; + prev->next = chunk; + break; + } + } +} + +yyjson_alc *yyjson_alc_dyn_new(void) { + const yyjson_alc def = YYJSON_DEFAULT_ALC; + usize hdr_len = sizeof(yyjson_alc) + sizeof(dyn_ctx); + yyjson_alc *alc = (yyjson_alc *)def.malloc(def.ctx, hdr_len); + dyn_ctx *ctx = (dyn_ctx *)(void *)(alc + 1); + if (unlikely(!alc)) return NULL; + alc->malloc = dyn_malloc; + alc->realloc = dyn_realloc; + alc->free = dyn_free; + alc->ctx = alc + 1; + memset(ctx, 0, sizeof(*ctx)); + return alc; +} + +void yyjson_alc_dyn_free(yyjson_alc *alc) { + const yyjson_alc def = YYJSON_DEFAULT_ALC; + dyn_ctx *ctx = (dyn_ctx *)(void *)(alc + 1); + dyn_chunk *chunk, *next; + if (unlikely(!alc)) return; + for (chunk = ctx->free_list.next; chunk; chunk = next) { + next = chunk->next; + def.free(def.ctx, chunk); + } + for (chunk = ctx->used_list.next; chunk; chunk = next) { + next = chunk->next; + def.free(def.ctx, chunk); + } + def.free(def.ctx, alc); +} + + + /*============================================================================== * JSON document and value *============================================================================*/ @@ -1164,48 +1338,77 @@ static_inline void unsafe_yyjson_val_pool_release(yyjson_val_pool *pool, } bool unsafe_yyjson_str_pool_grow(yyjson_str_pool *pool, - yyjson_alc *alc, usize len) { + const yyjson_alc *alc, usize len) { yyjson_str_chunk *chunk; - usize size = len + sizeof(yyjson_str_chunk); + usize size, max_len; + + /* create a new chunk */ + max_len = USIZE_MAX - sizeof(yyjson_str_chunk); + if (unlikely(len > max_len)) return false; + size = len + sizeof(yyjson_str_chunk); size = yyjson_max(pool->chunk_size, size); chunk = (yyjson_str_chunk *)alc->malloc(alc->ctx, size); - if (yyjson_unlikely(!chunk)) return false; + if (unlikely(!chunk)) return false; + /* insert the new chunk as the head of the linked list */ chunk->next = pool->chunks; + chunk->chunk_size = size; pool->chunks = chunk; pool->cur = (char *)chunk + sizeof(yyjson_str_chunk); pool->end = (char *)chunk + size; + /* the next chunk is twice the size of the current one */ size = yyjson_min(pool->chunk_size * 2, pool->chunk_size_max); + if (size < pool->chunk_size) size = pool->chunk_size_max; /* overflow */ pool->chunk_size = size; return true; } bool unsafe_yyjson_val_pool_grow(yyjson_val_pool *pool, - yyjson_alc *alc, usize count) { + const yyjson_alc *alc, usize count) { yyjson_val_chunk *chunk; - usize size; + usize size, max_count; - if (count >= USIZE_MAX / sizeof(yyjson_mut_val) - 16) return false; + /* create a new chunk */ + max_count = USIZE_MAX / sizeof(yyjson_mut_val) - 1; + if (unlikely(count > max_count)) return false; size = (count + 1) * sizeof(yyjson_mut_val); size = yyjson_max(pool->chunk_size, size); chunk = (yyjson_val_chunk *)alc->malloc(alc->ctx, size); - if (yyjson_unlikely(!chunk)) return false; + if (unlikely(!chunk)) return false; + /* insert the new chunk as the head of the linked list */ chunk->next = pool->chunks; + chunk->chunk_size = size; pool->chunks = chunk; - pool->cur = (yyjson_mut_val *)(void *)((u8 *)chunk - + sizeof(yyjson_mut_val)); + pool->cur = (yyjson_mut_val *)(void *)((u8 *)chunk) + 1; pool->end = (yyjson_mut_val *)(void *)((u8 *)chunk + size); + /* the next chunk is twice the size of the current one */ size = yyjson_min(pool->chunk_size * 2, pool->chunk_size_max); + if (size < pool->chunk_size) size = pool->chunk_size_max; /* overflow */ pool->chunk_size = size; return true; } +bool yyjson_mut_doc_set_str_pool_size(yyjson_mut_doc *doc, size_t len) { + usize max_size = USIZE_MAX - sizeof(yyjson_str_chunk); + if (!doc || !len || len > max_size) return false; + doc->str_pool.chunk_size = len + sizeof(yyjson_str_chunk); + return true; +} + +bool yyjson_mut_doc_set_val_pool_size(yyjson_mut_doc *doc, size_t count) { + usize max_count = USIZE_MAX / sizeof(yyjson_mut_val) - 1; + if (!doc || !count || count > max_count) return false; + doc->val_pool.chunk_size = (count + 1) * sizeof(yyjson_mut_val); + return true; +} + void yyjson_mut_doc_free(yyjson_mut_doc *doc) { if (doc) { yyjson_alc alc = doc->alc; + memset(&doc->alc, 0, sizeof(alc)); unsafe_yyjson_str_pool_release(&doc->str_pool, &alc); unsafe_yyjson_val_pool_release(&doc->val_pool, &alc); alc.free(alc.ctx, doc); @@ -1220,15 +1423,14 @@ yyjson_mut_doc *yyjson_mut_doc_new(const yyjson_alc *alc) { memset(doc, 0, sizeof(yyjson_mut_doc)); doc->alc = *alc; - doc->str_pool.chunk_size = 0x100; - doc->str_pool.chunk_size_max = 0x10000000; - doc->val_pool.chunk_size = 0x10 * sizeof(yyjson_mut_val); - doc->val_pool.chunk_size_max = 0x1000000 * sizeof(yyjson_mut_val); + doc->str_pool.chunk_size = YYJSON_MUT_DOC_STR_POOL_INIT_SIZE; + doc->str_pool.chunk_size_max = YYJSON_MUT_DOC_STR_POOL_MAX_SIZE; + doc->val_pool.chunk_size = YYJSON_MUT_DOC_VAL_POOL_INIT_SIZE; + doc->val_pool.chunk_size_max = YYJSON_MUT_DOC_VAL_POOL_MAX_SIZE; return doc; } -yyjson_api yyjson_mut_doc *yyjson_doc_mut_copy(yyjson_doc *doc, - const yyjson_alc *alc) { +yyjson_mut_doc *yyjson_doc_mut_copy(yyjson_doc *doc, const yyjson_alc *alc) { yyjson_mut_doc *m_doc; yyjson_mut_val *m_val; @@ -1244,12 +1446,14 @@ yyjson_api yyjson_mut_doc *yyjson_doc_mut_copy(yyjson_doc *doc, return m_doc; } -yyjson_api yyjson_mut_doc *yyjson_mut_doc_mut_copy(yyjson_mut_doc *doc, - const yyjson_alc *alc) { +yyjson_mut_doc *yyjson_mut_doc_mut_copy(yyjson_mut_doc *doc, + const yyjson_alc *alc) { yyjson_mut_doc *m_doc; yyjson_mut_val *m_val; - if (!doc || !doc->root) return NULL; + if (!doc) return NULL; + if (!doc->root) return yyjson_mut_doc_new(alc); + m_doc = yyjson_mut_doc_new(alc); if (!m_doc) return NULL; m_val = yyjson_mut_val_mut_copy(m_doc, doc->root); @@ -1261,14 +1465,13 @@ yyjson_api yyjson_mut_doc *yyjson_mut_doc_mut_copy(yyjson_mut_doc *doc, return m_doc; } -yyjson_api yyjson_mut_val *yyjson_val_mut_copy(yyjson_mut_doc *m_doc, - yyjson_val *i_vals) { +yyjson_mut_val *yyjson_val_mut_copy(yyjson_mut_doc *m_doc, + yyjson_val *i_vals) { /* The immutable object or array stores all sub-values in a contiguous memory, We copy them to another contiguous memory as mutable values, then reconnect the mutable values with the original relationship. */ - usize i_vals_len; yyjson_mut_val *m_vals, *m_val; yyjson_val *i_val, *i_end; @@ -1338,7 +1541,6 @@ static yyjson_mut_val *unsafe_yyjson_mut_val_mut_copy(yyjson_mut_doc *m_doc, second to last item, which needs to be linked to the last item to close the circle. */ - yyjson_mut_val *m_val = unsafe_yyjson_mut_val(m_doc, 1); if (unlikely(!m_val)) return NULL; m_val->tag = m_vals->tag; @@ -1379,35 +1581,148 @@ static yyjson_mut_val *unsafe_yyjson_mut_val_mut_copy(yyjson_mut_doc *m_doc, return m_val; } -yyjson_api yyjson_mut_val *yyjson_mut_val_mut_copy(yyjson_mut_doc *doc, - yyjson_mut_val *val) { +yyjson_mut_val *yyjson_mut_val_mut_copy(yyjson_mut_doc *doc, + yyjson_mut_val *val) { if (doc && val) return unsafe_yyjson_mut_val_mut_copy(doc, val); return NULL; } +/* Count the number of values and the total length of the strings. */ +static void yyjson_mut_stat(yyjson_mut_val *val, + usize *val_sum, usize *str_sum) { + yyjson_type type = unsafe_yyjson_get_type(val); + *val_sum += 1; + if (type == YYJSON_TYPE_ARR || type == YYJSON_TYPE_OBJ) { + yyjson_mut_val *child = (yyjson_mut_val *)val->uni.ptr; + usize len = unsafe_yyjson_get_len(val), i; + len <<= (u8)(type == YYJSON_TYPE_OBJ); + *val_sum += len; + for (i = 0; i < len; i++) { + yyjson_type stype = unsafe_yyjson_get_type(child); + if (stype == YYJSON_TYPE_STR || stype == YYJSON_TYPE_RAW) { + *str_sum += unsafe_yyjson_get_len(child) + 1; + } else if (stype == YYJSON_TYPE_ARR || stype == YYJSON_TYPE_OBJ) { + yyjson_mut_stat(child, val_sum, str_sum); + *val_sum -= 1; + } + child = child->next; + } + } else if (type == YYJSON_TYPE_STR || type == YYJSON_TYPE_RAW) { + *str_sum += unsafe_yyjson_get_len(val) + 1; + } +} + +/* Copy mutable values to immutable value pool. */ +static usize yyjson_imut_copy(yyjson_val **val_ptr, char **buf_ptr, + yyjson_mut_val *mval) { + yyjson_val *val = *val_ptr; + yyjson_type type = unsafe_yyjson_get_type(mval); + if (type == YYJSON_TYPE_ARR || type == YYJSON_TYPE_OBJ) { + yyjson_mut_val *child = (yyjson_mut_val *)mval->uni.ptr; + usize len = unsafe_yyjson_get_len(mval), i; + usize val_sum = 1; + if (type == YYJSON_TYPE_OBJ) { + if (len) child = child->next->next; + len <<= 1; + } else { + if (len) child = child->next; + } + *val_ptr = val + 1; + for (i = 0; i < len; i++) { + val_sum += yyjson_imut_copy(val_ptr, buf_ptr, child); + child = child->next; + } + val->tag = mval->tag; + val->uni.ofs = val_sum * sizeof(yyjson_val); + return val_sum; + } else if (type == YYJSON_TYPE_STR || type == YYJSON_TYPE_RAW) { + char *buf = *buf_ptr; + usize len = unsafe_yyjson_get_len(mval); + memcpy((void *)buf, (const void *)mval->uni.str, len); + buf[len] = '\0'; + val->tag = mval->tag; + val->uni.str = buf; + *val_ptr = val + 1; + *buf_ptr = buf + len + 1; + return 1; + } else { + val->tag = mval->tag; + val->uni = mval->uni; + *val_ptr = val + 1; + return 1; + } +} + +yyjson_doc *yyjson_mut_doc_imut_copy(yyjson_mut_doc *mdoc, + const yyjson_alc *alc) { + if (!mdoc) return NULL; + return yyjson_mut_val_imut_copy(mdoc->root, alc); +} + +yyjson_doc *yyjson_mut_val_imut_copy(yyjson_mut_val *mval, + const yyjson_alc *alc) { + usize val_num = 0, str_sum = 0, hdr_size, buf_size; + yyjson_doc *doc = NULL; + yyjson_val *val_hdr = NULL; + + /* This value should be NULL here. Setting a non-null value suppresses + warning from the clang analyzer. */ + char *str_hdr = (char *)(void *)&str_sum; + if (!mval) return NULL; + if (!alc) alc = &YYJSON_DEFAULT_ALC; + + /* traverse the input value to get pool size */ + yyjson_mut_stat(mval, &val_num, &str_sum); + + /* create doc and val pool */ + hdr_size = size_align_up(sizeof(yyjson_doc), sizeof(yyjson_val)); + buf_size = hdr_size + val_num * sizeof(yyjson_val); + doc = (yyjson_doc *)alc->malloc(alc->ctx, buf_size); + if (!doc) return NULL; + memset(doc, 0, sizeof(yyjson_doc)); + val_hdr = (yyjson_val *)(void *)((char *)(void *)doc + hdr_size); + doc->root = val_hdr; + doc->alc = *alc; + + /* create str pool */ + if (str_sum > 0) { + str_hdr = (char *)alc->malloc(alc->ctx, str_sum); + doc->str_pool = str_hdr; + if (!str_hdr) { + alc->free(alc->ctx, (void *)doc); + return NULL; + } + } + + /* copy vals and strs */ + doc->val_read = yyjson_imut_copy(&val_hdr, &str_hdr, mval); + doc->dat_read = str_sum + 1; + return doc; +} + static_inline bool unsafe_yyjson_num_equals(void *lhs, void *rhs) { yyjson_val_uni *luni = &((yyjson_val *)lhs)->uni; yyjson_val_uni *runi = &((yyjson_val *)rhs)->uni; yyjson_subtype lt = unsafe_yyjson_get_subtype(lhs); yyjson_subtype rt = unsafe_yyjson_get_subtype(rhs); - if (lt == rt) - return luni->u64 == runi->u64; - if (lt == YYJSON_SUBTYPE_SINT && rt == YYJSON_SUBTYPE_UINT) + if (lt == rt) return luni->u64 == runi->u64; + if (lt == YYJSON_SUBTYPE_SINT && rt == YYJSON_SUBTYPE_UINT) { return luni->i64 >= 0 && luni->u64 == runi->u64; - if (lt == YYJSON_SUBTYPE_UINT && rt == YYJSON_SUBTYPE_SINT) + } + if (lt == YYJSON_SUBTYPE_UINT && rt == YYJSON_SUBTYPE_SINT) { return runi->i64 >= 0 && luni->u64 == runi->u64; + } return false; } static_inline bool unsafe_yyjson_str_equals(void *lhs, void *rhs) { usize len = unsafe_yyjson_get_len(lhs); if (len != unsafe_yyjson_get_len(rhs)) return false; - return 0 == len || - 0 == memcmp(unsafe_yyjson_get_str(lhs), - unsafe_yyjson_get_str(rhs), len); + return !memcmp(unsafe_yyjson_get_str(lhs), + unsafe_yyjson_get_str(rhs), len); } -yyjson_api bool unsafe_yyjson_equals(yyjson_val *lhs, yyjson_val *rhs) { +bool unsafe_yyjson_equals(yyjson_val *lhs, yyjson_val *rhs) { yyjson_type type = unsafe_yyjson_get_type(lhs); if (type != unsafe_yyjson_get_type(rhs)) return false; @@ -1422,8 +1737,8 @@ yyjson_api bool unsafe_yyjson_equals(yyjson_val *lhs, yyjson_val *rhs) { while (len-- > 0) { rhs = yyjson_obj_iter_getn(&iter, lhs->uni.str, unsafe_yyjson_get_len(lhs)); - if (!rhs || !unsafe_yyjson_equals(lhs + 1, rhs)) - return false; + if (!rhs) return false; + if (!unsafe_yyjson_equals(lhs + 1, rhs)) return false; lhs = unsafe_yyjson_get_next(lhs + 1); } } @@ -1438,8 +1753,7 @@ yyjson_api bool unsafe_yyjson_equals(yyjson_val *lhs, yyjson_val *rhs) { lhs = unsafe_yyjson_get_first(lhs); rhs = unsafe_yyjson_get_first(rhs); while (len-- > 0) { - if (!unsafe_yyjson_equals(lhs, rhs)) - return false; + if (!unsafe_yyjson_equals(lhs, rhs)) return false; lhs = unsafe_yyjson_get_next(lhs); rhs = unsafe_yyjson_get_next(rhs); } @@ -1478,8 +1792,8 @@ bool unsafe_yyjson_mut_equals(yyjson_mut_val *lhs, yyjson_mut_val *rhs) { while (len-- > 0) { rhs = yyjson_mut_obj_iter_getn(&iter, lhs->uni.str, unsafe_yyjson_get_len(lhs)); - if (!rhs || !unsafe_yyjson_mut_equals(lhs->next, rhs)) - return false; + if (!rhs) return false; + if (!unsafe_yyjson_mut_equals(lhs->next, rhs)) return false; lhs = lhs->next->next; } } @@ -1494,8 +1808,7 @@ bool unsafe_yyjson_mut_equals(yyjson_mut_val *lhs, yyjson_mut_val *rhs) { lhs = (yyjson_mut_val *)lhs->uni.ptr; rhs = (yyjson_mut_val *)rhs->uni.ptr; while (len-- > 0) { - if (!unsafe_yyjson_mut_equals(lhs, rhs)) - return false; + if (!unsafe_yyjson_mut_equals(lhs, rhs)) return false; lhs = lhs->next; rhs = rhs->next; } @@ -1521,187 +1834,833 @@ bool unsafe_yyjson_mut_equals(yyjson_mut_val *lhs, yyjson_mut_val *rhs) { +#if !YYJSON_DISABLE_UTILS + /*============================================================================== - * JSON Pointer + * JSON Pointer API (RFC 6901) *============================================================================*/ /** - Get value from JSON array with a path segment (array index). - @param ptr Input the segment after `/`, output the end of segment. - @param end The end of entire JSON pointer. - @param arr JSON array (yyjson_val/yyjson_mut_val, based on `mut`). - @param mut Whether `arr` is mutable. - @return The matched value, or NULL if not matched. + Get a token from JSON pointer string. + @param ptr [in,out] + in: string that points to current token prefix `/` + out: string that points to next token prefix `/`, or string end + @param end [in] end of the entire JSON Pointer string + @param len [out] unescaped token length + @param esc [out] number of escaped characters in this token + @return head of the token, or NULL if syntax error */ -static_inline void *pointer_read_arr(const char **ptr, - const char *end, - void *arr, - bool mut) { - const char *hdr = *ptr; +static_inline const char *ptr_next_token(const char **ptr, const char *end, + usize *len, usize *esc) { + const char *hdr = *ptr + 1; const char *cur = hdr; - yyjson_val *i_arr = (yyjson_val *)arr; - yyjson_mut_val *m_arr = (yyjson_mut_val *)arr; - u64 idx = 0; - u8 add; - - /* start with 0 */ - if (cur < end && *cur == '0') { - *ptr = cur + 1; - return mut - ? (void *)yyjson_mut_arr_get_first(m_arr) - : (void *)yyjson_arr_get_first(i_arr); - } - - /* read whole number */ - if (cur + U64_SAFE_DIG < end) end = cur + U64_SAFE_DIG; - while (cur < end && (add = (u8)((u8)*cur - (u8)'0')) <= 9) { - cur++; - idx = idx * 10 + add; + /* skip unescaped characters */ + while (cur < end && *cur != '/' && *cur != '~') cur++; + if (likely(cur == end || *cur != '~')) { + /* no escaped characters, return */ + *ptr = cur; + *len = (usize)(cur - hdr); + *esc = 0; + return hdr; + } else { + /* handle escaped characters */ + usize esc_num = 0; + while (cur < end && *cur != '/') { + if (*cur++ == '~') { + if (cur == end || (*cur != '0' && *cur != '1')) { + *ptr = cur - 1; + return NULL; + } + esc_num++; + } + } + *ptr = cur; + *len = (usize)(cur - hdr) - esc_num; + *esc = esc_num; + return hdr; } - if (cur == hdr || idx >= (u64)USIZE_MAX) return NULL; - *ptr = cur; - return mut - ? (void *)yyjson_mut_arr_get(m_arr, (usize)idx) - : (void *)yyjson_arr_get(i_arr, (usize)idx); } /** - Get value from JSON object with a path segment (object key). - @param ptr Input the segment after `/`, output the end of segment. - @param end The end of entire JSON pointer. - @param obj JSON object (yyjson_val/yyjson_mut_val, based on `mut`). - @param mut `obj` is mutable. - @return The matched value, or NULL if not matched. + Convert token string to index. + @param cur [in] token head + @param len [in] token length + @param idx [out] the index number, or USIZE_MAX if token is '-' + @return true if token is a valid array index */ -static_inline void *pointer_read_obj(const char **ptr, - const char *end, - void *obj, - bool mut) { -#define BUF_SIZE 512 -#define is_escaped(cur) ((cur) < end && (*(cur) == '0' || *(cur) == '1')) -#define is_unescaped(cur) ((cur) < end && *(cur) != '/' && *(cur) != '~') -#define is_completed(cur) ((cur) == end || *(cur) == '/') - - const char *hdr = *ptr; - const char *cur = hdr; - yyjson_val *i_obj = (yyjson_val *)obj; - yyjson_mut_val *m_obj = (yyjson_mut_val *)obj; - yyjson_obj_iter i_iter; - yyjson_mut_obj_iter m_iter; - void *key; +static_inline bool ptr_token_to_idx(const char *cur, usize len, usize *idx) { + const char *end = cur + len; + usize num = 0, add; + if (unlikely(len == 0 || len > USIZE_SAFE_DIG)) return false; + if (*cur == '0') { + if (unlikely(len > 1)) return false; + *idx = 0; + return true; + } + if (*cur == '-') { + if (unlikely(len > 1)) return false; + *idx = USIZE_MAX; + return true; + } + for (; cur < end && (add = (usize)((u8)*cur - (u8)'0')) <= 9; cur++) { + num = num * 10 + add; + } + if (unlikely(num == 0 || cur < end)) return false; + *idx = num; + return true; +} + +/** + Compare JSON key with token. + @param key a string key (yyjson_val or yyjson_mut_val) + @param token a JSON pointer token + @param len unescaped token length + @param esc number of escaped characters in this token + @return true if `str` is equals to `token` + */ +static_inline bool ptr_token_eq(void *key, + const char *token, usize len, usize esc) { + yyjson_val *val = (yyjson_val *)key; + if (unsafe_yyjson_get_len(val) != len) return false; + if (likely(!esc)) { + return memcmp(val->uni.str, token, len) == 0; + } else { + const char *str = val->uni.str; + for (; len-- > 0; token++, str++) { + if (*token == '~') { + if (*str != (*++token == '0' ? '~' : '/')) return false; + } else { + if (*str != *token) return false; + } + } + return true; + } +} + +/** + Get a value from array by token. + @param arr an array, should not be NULL or non-array type + @param token a JSON pointer token + @param len unescaped token length + @param esc number of escaped characters in this token + @return value at index, or NULL if token is not index or index is out of range + */ +static_inline yyjson_val *ptr_arr_get(yyjson_val *arr, const char *token, + usize len, usize esc) { + yyjson_val *val = unsafe_yyjson_get_first(arr); + usize num = unsafe_yyjson_get_len(arr), idx = 0; + if (unlikely(num == 0)) return NULL; + if (unlikely(!ptr_token_to_idx(token, len, &idx))) return NULL; + if (unlikely(idx >= num)) return NULL; + if (unsafe_yyjson_arr_is_flat(arr)) { + return val + idx; + } else { + while (idx-- > 0) val = unsafe_yyjson_get_next(val); + return val; + } +} + +/** + Get a value from object by token. + @param obj [in] an object, should not be NULL or non-object type + @param token [in] a JSON pointer token + @param len [in] unescaped token length + @param esc [in] number of escaped characters in this token + @return value associated with the token, or NULL if no value + */ +static_inline yyjson_val *ptr_obj_get(yyjson_val *obj, const char *token, + usize len, usize esc) { + yyjson_val *key = unsafe_yyjson_get_first(obj); + usize num = unsafe_yyjson_get_len(obj); + if (unlikely(num == 0)) return NULL; + for (; num > 0; num--, key = unsafe_yyjson_get_next(key + 1)) { + if (ptr_token_eq(key, token, len, esc)) return key + 1; + } + return NULL; +} + +/** + Get a value from array by token. + @param arr [in] an array, should not be NULL or non-array type + @param token [in] a JSON pointer token + @param len [in] unescaped token length + @param esc [in] number of escaped characters in this token + @param pre [out] previous (sibling) value of the returned value + @param last [out] whether index is last + @return value at index, or NULL if token is not index or index is out of range + */ +static_inline yyjson_mut_val *ptr_mut_arr_get(yyjson_mut_val *arr, + const char *token, + usize len, usize esc, + yyjson_mut_val **pre, + bool *last) { + yyjson_mut_val *val = (yyjson_mut_val *)arr->uni.ptr; /* last (tail) */ + usize num = unsafe_yyjson_get_len(arr), idx; + if (last) *last = false; + if (false) *pre = NULL; + if (unlikely(num == 0)) { + if (last && len == 1 && (*token == '0' || *token == '-')) *last = true; + return NULL; + } + if (unlikely(!ptr_token_to_idx(token, len, &idx))) return NULL; + if (last) *last = (idx == num || idx == USIZE_MAX); + if (unlikely(idx >= num)) return NULL; + while (idx-- > 0) val = val->next; + *pre = val; + return val->next; +} + +/** + Get a value from object by token. + @param obj [in] an object, should not be NULL or non-object type + @param token [in] a JSON pointer token + @param len [in] unescaped token length + @param esc [in] number of escaped characters in this token + @param pre [out] previous (sibling) key of the returned value's key + @return value associated with the token, or NULL if no value + */ +static_inline yyjson_mut_val *ptr_mut_obj_get(yyjson_mut_val *obj, + const char *token, + usize len, usize esc, + yyjson_mut_val **pre) { + yyjson_mut_val *pre_key = (yyjson_mut_val *)obj->uni.ptr, *key; + usize num = unsafe_yyjson_get_len(obj); + if (false) *pre = NULL; + if (unlikely(num == 0)) return NULL; + for (; num > 0; num--, pre_key = key) { + key = pre_key->next->next; + if (ptr_token_eq(key, token, len, esc)) { + *pre = pre_key; + return key->next; + } + } + return NULL; +} + +/** + Create a string value with JSON pointer token. + @param token [in] a JSON pointer token + @param len [in] unescaped token length + @param esc [in] number of escaped characters in this token + @param doc [in] used for memory allocation when creating value + @return new string value, or NULL if memory allocation failed + */ +static_inline yyjson_mut_val *ptr_new_key(const char *token, + usize len, usize esc, + yyjson_mut_doc *doc) { + const char *src = token; + if (likely(!esc)) { + return yyjson_mut_strncpy(doc, src, len); + } else { + const char *end = src + len + esc; + char *dst = unsafe_yyjson_mut_str_alc(doc, len + esc); + char *str = dst; + if (unlikely(!dst)) return NULL; + for (; src < end; src++, dst++) { + if (*src != '~') *dst = *src; + else *dst = (*++src == '0' ? '~' : '/'); + } + *dst = '\0'; + return yyjson_mut_strn(doc, str, len); + } +} + +/* macros for yyjson_ptr */ +#define return_err(_ret, _code, _pos, _msg) do { \ + if (err) { \ + err->code = YYJSON_PTR_ERR_##_code; \ + err->msg = _msg; \ + err->pos = (usize)(_pos); \ + } \ + return _ret; \ +} while (false) + +#define return_err_resolve(_ret, _pos) \ + return_err(_ret, RESOLVE, _pos, "JSON pointer cannot be resolved") +#define return_err_syntax(_ret, _pos) \ + return_err(_ret, SYNTAX, _pos, "invalid escaped character") +#define return_err_alloc(_ret) \ + return_err(_ret, MEMORY_ALLOCATION, 0, "failed to create value") + +yyjson_val *unsafe_yyjson_ptr_getx(yyjson_val *val, + const char *ptr, size_t ptr_len, + yyjson_ptr_err *err) { - /* skip unescaped characters */ - while (is_unescaped(cur)) cur++; - if (likely(is_completed(cur))) { - usize len = (usize)(cur - hdr); - *ptr = cur; - return mut - ? (void *)yyjson_mut_obj_getn(m_obj, hdr, len) - : (void *)yyjson_obj_getn(i_obj, hdr, len); + const char *hdr = ptr, *end = ptr + ptr_len, *token; + usize len, esc; + yyjson_type type; + + while (true) { + token = ptr_next_token(&ptr, end, &len, &esc); + if (unlikely(!token)) return_err_syntax(NULL, ptr - hdr); + type = unsafe_yyjson_get_type(val); + if (type == YYJSON_TYPE_OBJ) { + val = ptr_obj_get(val, token, len, esc); + } else if (type == YYJSON_TYPE_ARR) { + val = ptr_arr_get(val, token, len, esc); + } else { + val = NULL; + } + if (!val) return_err_resolve(NULL, token - hdr); + if (ptr == end) return val; } +} + +yyjson_mut_val *unsafe_yyjson_mut_ptr_getx(yyjson_mut_val *val, + const char *ptr, + size_t ptr_len, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err) { - /* copy escaped characters to buffer */ - if (likely(end - hdr <= BUF_SIZE)) { - char buf[BUF_SIZE]; - char *dst = buf + (usize)(cur - hdr); - memcpy(buf, hdr, (usize)(cur - hdr)); - while (true) { - if (is_unescaped(cur)) { - *dst++ = *cur++; - } else if (is_completed(cur)) { - usize len = (usize)(dst - buf); - *ptr = cur; - return mut - ? (void *)yyjson_mut_obj_getn(m_obj, buf, len) - : (void *)yyjson_obj_getn(i_obj, buf, len); + const char *hdr = ptr, *end = ptr + ptr_len, *token; + usize len, esc; + yyjson_mut_val *ctn, *pre = NULL; + yyjson_type type; + bool idx_is_last = false; + + while (true) { + token = ptr_next_token(&ptr, end, &len, &esc); + if (unlikely(!token)) return_err_syntax(NULL, ptr - hdr); + ctn = val; + type = unsafe_yyjson_get_type(val); + if (type == YYJSON_TYPE_OBJ) { + val = ptr_mut_obj_get(val, token, len, esc, &pre); + } else if (type == YYJSON_TYPE_ARR) { + val = ptr_mut_arr_get(val, token, len, esc, &pre, &idx_is_last); + } else { + val = NULL; + } + if (ctx && (ptr == end)) { + if (type == YYJSON_TYPE_OBJ || + (type == YYJSON_TYPE_ARR && (val || idx_is_last))) { + ctx->ctn = ctn; + ctx->pre = pre; + } + } + if (!val) return_err_resolve(NULL, token - hdr); + if (ptr == end) return val; + } +} + +bool unsafe_yyjson_mut_ptr_putx(yyjson_mut_val *val, + const char *ptr, size_t ptr_len, + yyjson_mut_val *new_val, + yyjson_mut_doc *doc, + bool create_parent, bool insert_new, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err) { + + const char *hdr = ptr, *end = ptr + ptr_len, *token; + usize token_len, esc, ctn_len; + yyjson_mut_val *ctn, *key, *pre = NULL; + yyjson_mut_val *sep_ctn = NULL, *sep_key = NULL, *sep_val = NULL; + yyjson_type ctn_type; + bool idx_is_last = false; + + /* skip exist parent nodes */ + while (true) { + token = ptr_next_token(&ptr, end, &token_len, &esc); + if (unlikely(!token)) return_err_syntax(false, ptr - hdr); + ctn = val; + ctn_type = unsafe_yyjson_get_type(ctn); + if (ctn_type == YYJSON_TYPE_OBJ) { + val = ptr_mut_obj_get(ctn, token, token_len, esc, &pre); + } else if (ctn_type == YYJSON_TYPE_ARR) { + val = ptr_mut_arr_get(ctn, token, token_len, esc, &pre, + &idx_is_last); + } else return_err_resolve(false, token - hdr); + if (!val) break; + if (ptr == end) break; /* is last token */ + } + + /* create parent nodes if not exist */ + if (unlikely(ptr != end)) { /* not last token */ + if (!create_parent) return_err_resolve(false, token - hdr); + + /* add value at last index if container is array */ + if (ctn_type == YYJSON_TYPE_ARR) { + if (!idx_is_last || !insert_new) { + return_err_resolve(false, token - hdr); + } + val = yyjson_mut_obj(doc); + if (!val) return_err_alloc(false); + + /* delay attaching until all operations are completed */ + sep_ctn = ctn; + sep_key = NULL; + sep_val = val; + + /* move to next token */ + ctn = val; + val = NULL; + ctn_type = YYJSON_TYPE_OBJ; + token = ptr_next_token(&ptr, end, &token_len, &esc); + if (unlikely(!token)) return_err_resolve(false, token - hdr); + } + + /* container is object, create parent nodes */ + while (ptr != end) { /* not last token */ + key = ptr_new_key(token, token_len, esc, doc); + if (!key) return_err_alloc(false); + val = yyjson_mut_obj(doc); + if (!val) return_err_alloc(false); + + /* delay attaching until all operations are completed */ + if (!sep_ctn) { + sep_ctn = ctn; + sep_key = key; + sep_val = val; } else { - cur++; /* skip '~' */ - if (unlikely(!is_escaped(cur))) return NULL; - *dst++ = (char)(*cur++ == '0' ? '~' : '/'); + yyjson_mut_obj_add(ctn, key, val); } + + /* move to next token */ + ctn = val; + val = NULL; + token = ptr_next_token(&ptr, end, &token_len, &esc); + if (unlikely(!token)) return_err_syntax(false, ptr - hdr); } } - /* compare byte by byte */ - cur = hdr; - if (!mut) yyjson_obj_iter_init(i_obj, &i_iter); - else yyjson_mut_obj_iter_init(m_obj, &m_iter); - while ((key = mut ? (void *)yyjson_mut_obj_iter_next(&m_iter) - : (void *)yyjson_obj_iter_next(&i_iter))) { - const char *k_str = unsafe_yyjson_get_str(key); - const char *k_end = k_str + unsafe_yyjson_get_len(key); - while (k_str < k_end) { - if (is_unescaped(cur) && *k_str == *cur) { - k_str += 1; - cur += 1; - } else if (cur < end && *cur == '~' && is_escaped(cur + 1) && - *k_str == (*(cur + 1) == '0' ? '~' : '/')) { - k_str += 1; - cur += 2; + /* JSON pointer is resolved, insert or replace target value */ + ctn_len = unsafe_yyjson_get_len(ctn); + if (ctn_type == YYJSON_TYPE_OBJ) { + if (ctx) ctx->ctn = ctn; + if (!val || insert_new) { + /* insert new key-value pair */ + key = ptr_new_key(token, token_len, esc, doc); + if (unlikely(!key)) return_err_alloc(false); + if (ctx) ctx->pre = ctn_len ? (yyjson_mut_val *)ctn->uni.ptr : key; + unsafe_yyjson_mut_obj_add(ctn, key, new_val, ctn_len); + } else { + /* replace exist value */ + key = pre->next->next; + if (ctx) ctx->pre = pre; + if (ctx) ctx->old = val; + yyjson_mut_obj_put(ctn, key, new_val); + } + } else { + /* array */ + if (ctx && (val || idx_is_last)) ctx->ctn = ctn; + if (insert_new) { + /* append new value */ + if (val) { + pre->next = new_val; + new_val->next = val; + if (ctx) ctx->pre = pre; + unsafe_yyjson_set_len(ctn, ctn_len + 1); + } else if (idx_is_last) { + if (ctx) ctx->pre = ctn_len ? + (yyjson_mut_val *)ctn->uni.ptr : new_val; + yyjson_mut_arr_append(ctn, new_val); } else { + return_err_resolve(false, token - hdr); + } + } else { + /* replace exist value */ + if (!val) return_err_resolve(false, token - hdr); + if (ctn_len > 1) { + new_val->next = val->next; + pre->next = new_val; + if (ctn->uni.ptr == val) ctn->uni.ptr = new_val; + } else { + new_val->next = new_val; + ctn->uni.ptr = new_val; + pre = new_val; + } + if (ctx) ctx->pre = pre; + if (ctx) ctx->old = val; + } + } + + /* all operations are completed, attach the new components to the target */ + if (unlikely(sep_ctn)) { + if (sep_key) yyjson_mut_obj_add(sep_ctn, sep_key, sep_val); + else yyjson_mut_arr_append(sep_ctn, sep_val); + } + return true; +} + +yyjson_mut_val *unsafe_yyjson_mut_ptr_replacex( + yyjson_mut_val *val, const char *ptr, size_t len, yyjson_mut_val *new_val, + yyjson_ptr_ctx *ctx, yyjson_ptr_err *err) { + + yyjson_mut_val *cur_val; + yyjson_ptr_ctx cur_ctx; + memset(&cur_ctx, 0, sizeof(cur_ctx)); + if (!ctx) ctx = &cur_ctx; + cur_val = unsafe_yyjson_mut_ptr_getx(val, ptr, len, ctx, err); + if (!cur_val) return NULL; + + if (yyjson_mut_is_obj(ctx->ctn)) { + yyjson_mut_val *key = ctx->pre->next->next; + yyjson_mut_obj_put(ctx->ctn, key, new_val); + } else { + yyjson_ptr_ctx_replace(ctx, new_val); + } + ctx->old = cur_val; + return cur_val; +} + +yyjson_mut_val *unsafe_yyjson_mut_ptr_removex(yyjson_mut_val *val, + const char *ptr, + size_t len, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err) { + yyjson_mut_val *cur_val; + yyjson_ptr_ctx cur_ctx; + memset(&cur_ctx, 0, sizeof(cur_ctx)); + if (!ctx) ctx = &cur_ctx; + cur_val = unsafe_yyjson_mut_ptr_getx(val, ptr, len, ctx, err); + if (cur_val) { + if (yyjson_mut_is_obj(ctx->ctn)) { + yyjson_mut_val *key = ctx->pre->next->next; + yyjson_mut_obj_put(ctx->ctn, key, NULL); + } else { + yyjson_ptr_ctx_remove(ctx); + } + ctx->pre = NULL; + ctx->old = cur_val; + } + return cur_val; +} + +/* macros for yyjson_ptr */ +#undef return_err +#undef return_err_resolve +#undef return_err_syntax +#undef return_err_alloc + + + +/*============================================================================== + * JSON Patch API (RFC 6902) + *============================================================================*/ + +/* JSON Patch operation */ +typedef enum patch_op { + PATCH_OP_ADD, /* path, value */ + PATCH_OP_REMOVE, /* path */ + PATCH_OP_REPLACE, /* path, value */ + PATCH_OP_MOVE, /* from, path */ + PATCH_OP_COPY, /* from, path */ + PATCH_OP_TEST, /* path, value */ + PATCH_OP_NONE /* invalid */ +} patch_op; + +static patch_op patch_op_get(yyjson_val *op) { + const char *str = op->uni.str; + switch (unsafe_yyjson_get_len(op)) { + case 3: + if (!memcmp(str, "add", 3)) return PATCH_OP_ADD; + return PATCH_OP_NONE; + case 4: + if (!memcmp(str, "move", 4)) return PATCH_OP_MOVE; + if (!memcmp(str, "copy", 4)) return PATCH_OP_COPY; + if (!memcmp(str, "test", 4)) return PATCH_OP_TEST; + return PATCH_OP_NONE; + case 6: + if (!memcmp(str, "remove", 6)) return PATCH_OP_REMOVE; + return PATCH_OP_NONE; + case 7: + if (!memcmp(str, "replace", 7)) return PATCH_OP_REPLACE; + return PATCH_OP_NONE; + default: + return PATCH_OP_NONE; + } +} + +/* macros for yyjson_patch */ +#define return_err(_code, _msg) do { \ + if (err->ptr.code == YYJSON_PTR_ERR_MEMORY_ALLOCATION) { \ + err->code = YYJSON_PATCH_ERROR_MEMORY_ALLOCATION; \ + err->msg = _msg; \ + memset(&err->ptr, 0, sizeof(yyjson_ptr_err)); \ + } else { \ + err->code = YYJSON_PATCH_ERROR_##_code; \ + err->msg = _msg; \ + err->idx = iter.idx ? iter.idx - 1 : 0; \ + } \ + return NULL; \ +} while (false) + +#define return_err_copy() \ + return_err(MEMORY_ALLOCATION, "failed to copy value") +#define return_err_key(_key) \ + return_err(MISSING_KEY, "missing key " _key) +#define return_err_val(_key) \ + return_err(INVALID_MEMBER, "invalid member " _key) + +#define ptr_get(_ptr) yyjson_mut_ptr_getx( \ + root, _ptr->uni.str, _ptr##_len, NULL, &err->ptr) +#define ptr_add(_ptr, _val) yyjson_mut_ptr_addx( \ + root, _ptr->uni.str, _ptr##_len, _val, doc, false, NULL, &err->ptr) +#define ptr_remove(_ptr) yyjson_mut_ptr_removex( \ + root, _ptr->uni.str, _ptr##_len, NULL, &err->ptr) +#define ptr_replace(_ptr, _val)yyjson_mut_ptr_replacex( \ + root, _ptr->uni.str, _ptr##_len, _val, NULL, &err->ptr) + +yyjson_mut_val *yyjson_patch(yyjson_mut_doc *doc, + yyjson_val *orig, + yyjson_val *patch, + yyjson_patch_err *err) { + + yyjson_mut_val *root; + yyjson_val *obj; + yyjson_arr_iter iter; + yyjson_patch_err err_tmp; + if (!err) err = &err_tmp; + memset(err, 0, sizeof(*err)); + memset(&iter, 0, sizeof(iter)); + + if (unlikely(!doc || !orig || !patch)) { + return_err(INVALID_PARAMETER, "input parameter is NULL"); + } + if (unlikely(!yyjson_is_arr(patch))) { + return_err(INVALID_PARAMETER, "input patch is not array"); + } + root = yyjson_val_mut_copy(doc, orig); + if (unlikely(!root)) return_err_copy(); + + /* iterate through the patch array */ + yyjson_arr_iter_init(patch, &iter); + while ((obj = yyjson_arr_iter_next(&iter))) { + patch_op op_enum; + yyjson_val *op, *path, *from = NULL, *value; + yyjson_mut_val *val = NULL, *test; + usize path_len, from_len = 0; + if (unlikely(!unsafe_yyjson_is_obj(obj))) { + return_err(INVALID_OPERATION, "JSON patch operation is not object"); + } + + /* get required member: op */ + op = yyjson_obj_get(obj, "op"); + if (unlikely(!op)) return_err_key("`op`"); + if (unlikely(!yyjson_is_str(op))) return_err_val("`op`"); + op_enum = patch_op_get(op); + + /* get required member: path */ + path = yyjson_obj_get(obj, "path"); + if (unlikely(!path)) return_err_key("`path`"); + if (unlikely(!yyjson_is_str(path))) return_err_val("`path`"); + path_len = unsafe_yyjson_get_len(path); + + /* get required member: value, from */ + switch ((int)op_enum) { + case PATCH_OP_ADD: case PATCH_OP_REPLACE: case PATCH_OP_TEST: + value = yyjson_obj_get(obj, "value"); + if (unlikely(!value)) return_err_key("`value`"); + val = yyjson_val_mut_copy(doc, value); + if (unlikely(!val)) return_err_copy(); + break; + case PATCH_OP_MOVE: case PATCH_OP_COPY: + from = yyjson_obj_get(obj, "from"); + if (unlikely(!from)) return_err_key("`from`"); + if (unlikely(!yyjson_is_str(from))) return_err_val("`from`"); + from_len = unsafe_yyjson_get_len(from); + break; + default: break; - } } - if (k_str == k_end && is_completed(cur)) { - *ptr = cur; - return mut - ? (void *)yyjson_mut_obj_iter_get_val((yyjson_mut_val *)key) - : (void *)yyjson_obj_iter_get_val((yyjson_val *)key); + + /* perform an operation */ + switch ((int)op_enum) { + case PATCH_OP_ADD: /* add(path, val) */ + if (unlikely(path_len == 0)) { root = val; break; } + if (unlikely(!ptr_add(path, val))) { + return_err(POINTER, "failed to add `path`"); + } + break; + case PATCH_OP_REMOVE: /* remove(path) */ + if (unlikely(!ptr_remove(path))) { + return_err(POINTER, "failed to remove `path`"); + } + break; + case PATCH_OP_REPLACE: /* replace(path, val) */ + if (unlikely(path_len == 0)) { root = val; break; } + if (unlikely(!ptr_replace(path, val))) { + return_err(POINTER, "failed to replace `path`"); + } + break; + case PATCH_OP_MOVE: /* val = remove(from), add(path, val) */ + if (unlikely(from_len == 0 && path_len == 0)) break; + val = ptr_remove(from); + if (unlikely(!val)) { + return_err(POINTER, "failed to remove `from`"); + } + if (unlikely(path_len == 0)) { root = val; break; } + if (unlikely(!ptr_add(path, val))) { + return_err(POINTER, "failed to add `path`"); + } + break; + case PATCH_OP_COPY: /* val = get(from).copy, add(path, val) */ + val = ptr_get(from); + if (unlikely(!val)) { + return_err(POINTER, "failed to get `from`"); + } + if (unlikely(path_len == 0)) { root = val; break; } + val = yyjson_mut_val_mut_copy(doc, val); + if (unlikely(!val)) return_err_copy(); + if (unlikely(!ptr_add(path, val))) { + return_err(POINTER, "failed to add `path`"); + } + break; + case PATCH_OP_TEST: /* test = get(path), test.eq(val) */ + test = ptr_get(path); + if (unlikely(!test)) { + return_err(POINTER, "failed to get `path`"); + } + if (unlikely(!yyjson_mut_equals(val, test))) { + return_err(EQUAL, "failed to test equal"); + } + break; + default: + return_err(INVALID_MEMBER, "unsupported `op`"); } } - return NULL; - -#undef BUF_SIZE -#undef is_escaped -#undef is_unescaped -#undef is_completed -} - -yyjson_api yyjson_val *unsafe_yyjson_get_pointer(yyjson_val *val, - const char *ptr, - usize len) { - const char *end = ptr + len; - ptr++; /* skip '/' */ - while (true) { - if (yyjson_is_obj(val)) { - val = (yyjson_val *)pointer_read_obj(&ptr, end, val, false); - } else if (yyjson_is_arr(val)) { - val = (yyjson_val *)pointer_read_arr(&ptr, end, val, false); - } else { - val = NULL; + return root; +} + +yyjson_mut_val *yyjson_mut_patch(yyjson_mut_doc *doc, + yyjson_mut_val *orig, + yyjson_mut_val *patch, + yyjson_patch_err *err) { + yyjson_mut_val *root, *obj; + yyjson_mut_arr_iter iter; + yyjson_patch_err err_tmp; + if (!err) err = &err_tmp; + memset(err, 0, sizeof(*err)); + memset(&iter, 0, sizeof(iter)); + + if (unlikely(!doc || !orig || !patch)) { + return_err(INVALID_PARAMETER, "input parameter is NULL"); + } + if (unlikely(!yyjson_mut_is_arr(patch))) { + return_err(INVALID_PARAMETER, "input patch is not array"); + } + root = yyjson_mut_val_mut_copy(doc, orig); + if (unlikely(!root)) return_err_copy(); + + /* iterate through the patch array */ + yyjson_mut_arr_iter_init(patch, &iter); + while ((obj = yyjson_mut_arr_iter_next(&iter))) { + patch_op op_enum; + yyjson_mut_val *op, *path, *from = NULL, *value; + yyjson_mut_val *val = NULL, *test; + usize path_len, from_len = 0; + if (!unsafe_yyjson_is_obj(obj)) { + return_err(INVALID_OPERATION, "JSON patch operation is not object"); } - if (!val || ptr == end) return val; - if (*ptr++ != '/') return NULL; - } -} - -yyjson_api yyjson_mut_val *unsafe_yyjson_mut_get_pointer(yyjson_mut_val *val, - const char *ptr, - usize len) { - const char *end = ptr + len; - ptr++; /* skip '/' */ - while (true) { - if (yyjson_mut_is_obj(val)) { - val = (yyjson_mut_val *)pointer_read_obj(&ptr, end, val, true); - } else if (yyjson_mut_is_arr(val)) { - val = (yyjson_mut_val *)pointer_read_arr(&ptr, end, val, true); - } else { - val = NULL; + + /* get required member: op */ + op = yyjson_mut_obj_get(obj, "op"); + if (unlikely(!op)) return_err_key("`op`"); + if (unlikely(!yyjson_mut_is_str(op))) return_err_val("`op`"); + op_enum = patch_op_get((yyjson_val *)(void *)op); + + /* get required member: path */ + path = yyjson_mut_obj_get(obj, "path"); + if (unlikely(!path)) return_err_key("`path`"); + if (unlikely(!yyjson_mut_is_str(path))) return_err_val("`path`"); + path_len = unsafe_yyjson_get_len(path); + + /* get required member: value, from */ + switch ((int)op_enum) { + case PATCH_OP_ADD: case PATCH_OP_REPLACE: case PATCH_OP_TEST: + value = yyjson_mut_obj_get(obj, "value"); + if (unlikely(!value)) return_err_key("`value`"); + val = yyjson_mut_val_mut_copy(doc, value); + if (unlikely(!val)) return_err_copy(); + break; + case PATCH_OP_MOVE: case PATCH_OP_COPY: + from = yyjson_mut_obj_get(obj, "from"); + if (unlikely(!from)) return_err_key("`from`"); + if (unlikely(!yyjson_mut_is_str(from))) { + return_err_val("`from`"); + } + from_len = unsafe_yyjson_get_len(from); + break; + default: + break; + } + + /* perform an operation */ + switch ((int)op_enum) { + case PATCH_OP_ADD: /* add(path, val) */ + if (unlikely(path_len == 0)) { root = val; break; } + if (unlikely(!ptr_add(path, val))) { + return_err(POINTER, "failed to add `path`"); + } + break; + case PATCH_OP_REMOVE: /* remove(path) */ + if (unlikely(!ptr_remove(path))) { + return_err(POINTER, "failed to remove `path`"); + } + break; + case PATCH_OP_REPLACE: /* replace(path, val) */ + if (unlikely(path_len == 0)) { root = val; break; } + if (unlikely(!ptr_replace(path, val))) { + return_err(POINTER, "failed to replace `path`"); + } + break; + case PATCH_OP_MOVE: /* val = remove(from), add(path, val) */ + if (unlikely(from_len == 0 && path_len == 0)) break; + val = ptr_remove(from); + if (unlikely(!val)) { + return_err(POINTER, "failed to remove `from`"); + } + if (unlikely(path_len == 0)) { root = val; break; } + if (unlikely(!ptr_add(path, val))) { + return_err(POINTER, "failed to add `path`"); + } + break; + case PATCH_OP_COPY: /* val = get(from).copy, add(path, val) */ + val = ptr_get(from); + if (unlikely(!val)) { + return_err(POINTER, "failed to get `from`"); + } + if (unlikely(path_len == 0)) { root = val; break; } + val = yyjson_mut_val_mut_copy(doc, val); + if (unlikely(!val)) return_err_copy(); + if (unlikely(!ptr_add(path, val))) { + return_err(POINTER, "failed to add `path`"); + } + break; + case PATCH_OP_TEST: /* test = get(path), test.eq(val) */ + test = ptr_get(path); + if (unlikely(!test)) { + return_err(POINTER, "failed to get `path`"); + } + if (unlikely(!yyjson_mut_equals(val, test))) { + return_err(EQUAL, "failed to test equal"); + } + break; + default: + return_err(INVALID_MEMBER, "unsupported `op`"); } - if (!val || ptr == end) return val; - if (*ptr++ != '/') return NULL; } + return root; } +/* macros for yyjson_patch */ +#undef return_err +#undef return_err_copy +#undef return_err_key +#undef return_err_val +#undef ptr_get +#undef ptr_add +#undef ptr_remove +#undef ptr_replace + /*============================================================================== - * JSON Merge-Patch + * JSON Merge-Patch API (RFC 7386) *============================================================================*/ -yyjson_api yyjson_mut_val *yyjson_merge_patch(yyjson_mut_doc *doc, - yyjson_val *orig, - yyjson_val *patch) { +yyjson_mut_val *yyjson_merge_patch(yyjson_mut_doc *doc, + yyjson_val *orig, + yyjson_val *patch) { usize idx, max; yyjson_val *key, *orig_val, *patch_val, local_orig; yyjson_mut_val *builder, *mut_key, *mut_val, *merged_val; @@ -1713,12 +2672,27 @@ yyjson_api yyjson_mut_val *yyjson_merge_patch(yyjson_mut_doc *doc, builder = yyjson_mut_obj(doc); if (unlikely(!builder)) return NULL; + memset(&local_orig, 0, sizeof(local_orig)); if (!yyjson_is_obj(orig)) { orig = &local_orig; orig->tag = builder->tag; orig->uni = builder->uni; } + /* If orig is contributing, copy any items not modified by the patch */ + if (orig != &local_orig) { + yyjson_obj_foreach(orig, idx, max, key, orig_val) { + patch_val = yyjson_obj_getn(patch, + unsafe_yyjson_get_str(key), + unsafe_yyjson_get_len(key)); + if (!patch_val) { + mut_key = yyjson_val_mut_copy(doc, key); + mut_val = yyjson_val_mut_copy(doc, orig_val); + if (!yyjson_mut_obj_add(builder, mut_key, mut_val)) return NULL; + } + } + } + /* Merge items modified by the patch. */ yyjson_obj_foreach(patch, idx, max, key, patch_val) { /* null indicates the field is removed. */ @@ -1733,29 +2707,12 @@ yyjson_api yyjson_mut_val *yyjson_merge_patch(yyjson_mut_doc *doc, if (!yyjson_mut_obj_add(builder, mut_key, merged_val)) return NULL; } - /* Exit early, if orig is not contributing to the final result. */ - if (orig == &local_orig) { - return builder; - } - - /* Copy over any items that weren't modified by the patch. */ - yyjson_obj_foreach(orig, idx, max, key, orig_val) { - patch_val = yyjson_obj_getn(patch, - unsafe_yyjson_get_str(key), - unsafe_yyjson_get_len(key)); - if (!patch_val) { - mut_key = yyjson_val_mut_copy(doc, key); - mut_val = yyjson_val_mut_copy(doc, orig_val); - if (!yyjson_mut_obj_add(builder, mut_key, mut_val)) return NULL; - } - } - return builder; } -yyjson_api yyjson_mut_val *yyjson_mut_merge_patch(yyjson_mut_doc *doc, - yyjson_mut_val *orig, - yyjson_mut_val *patch) { +yyjson_mut_val *yyjson_mut_merge_patch(yyjson_mut_doc *doc, + yyjson_mut_val *orig, + yyjson_mut_val *patch) { usize idx, max; yyjson_mut_val *key, *orig_val, *patch_val, local_orig; yyjson_mut_val *builder, *mut_key, *mut_val, *merged_val; @@ -1767,12 +2724,27 @@ yyjson_api yyjson_mut_val *yyjson_mut_merge_patch(yyjson_mut_doc *doc, builder = yyjson_mut_obj(doc); if (unlikely(!builder)) return NULL; + memset(&local_orig, 0, sizeof(local_orig)); if (!yyjson_mut_is_obj(orig)) { orig = &local_orig; orig->tag = builder->tag; orig->uni = builder->uni; } + /* If orig is contributing, copy any items not modified by the patch */ + if (orig != &local_orig) { + yyjson_mut_obj_foreach(orig, idx, max, key, orig_val) { + patch_val = yyjson_mut_obj_getn(patch, + unsafe_yyjson_get_str(key), + unsafe_yyjson_get_len(key)); + if (!patch_val) { + mut_key = yyjson_mut_val_mut_copy(doc, key); + mut_val = yyjson_mut_val_mut_copy(doc, orig_val); + if (!yyjson_mut_obj_add(builder, mut_key, mut_val)) return NULL; + } + } + } + /* Merge items modified by the patch. */ yyjson_mut_obj_foreach(patch, idx, max, key, patch_val) { /* null indicates the field is removed. */ @@ -1787,26 +2759,11 @@ yyjson_api yyjson_mut_val *yyjson_mut_merge_patch(yyjson_mut_doc *doc, if (!yyjson_mut_obj_add(builder, mut_key, merged_val)) return NULL; } - /* Exit early, if orig is not contributing to the final result. */ - if (orig == &local_orig) { - return builder; - } - - /* Copy over any items that weren't modified by the patch. */ - yyjson_mut_obj_foreach(orig, idx, max, key, orig_val) { - patch_val = yyjson_mut_obj_getn(patch, - unsafe_yyjson_get_str(key), - unsafe_yyjson_get_len(key)); - if (!patch_val) { - mut_key = yyjson_mut_val_mut_copy(doc, key); - mut_val = yyjson_mut_val_mut_copy(doc, orig_val); - if (!yyjson_mut_obj_add(builder, mut_key, mut_val)) return NULL; - } - } - return builder; } +#endif /* YYJSON_DISABLE_UTILS */ + /*============================================================================== @@ -2529,8 +3486,6 @@ static_inline void pow10_table_get_exp(i32 exp10, i32 *exp2) { -#if !YYJSON_DISABLE_READER - /*============================================================================== * JSON Character Matcher *============================================================================*/ @@ -2556,9 +3511,12 @@ static const char_type CHAR_TYPE_CONTAINER = 1 << 4; /** Comment character: '/'. */ static const char_type CHAR_TYPE_COMMENT = 1 << 5; -/** Line end character '\\n', '\\r', '\0'. */ +/** Line end character: '\\n', '\\r', '\0'. */ static const char_type CHAR_TYPE_LINE_END = 1 << 6; +/** Hexadecimal numeric character: [0-9a-fA-F]. */ +static const char_type CHAR_TYPE_HEX = 1 << 7; + /** Character type table (generate with misc/make_tables.c) */ static const char_type char_table[256] = { 0x44, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, @@ -2567,13 +3525,13 @@ static const char_type char_table[256] = { 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x20, - 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, - 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x82, 0x82, 0x82, 0x82, 0x82, 0x82, 0x82, 0x82, + 0x82, 0x82, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, @@ -2620,17 +3578,22 @@ static_inline bool char_is_container(u8 c) { return char_is_type(c, (char_type)CHAR_TYPE_CONTAINER); } -/** Match a stop character in ASCII string: '"', '\', [0x00-0x1F], [0x80-0xFF]*/ +/** Match a stop character in ASCII string: '"', '\', [0x00-0x1F,0x80-0xFF]. */ static_inline bool char_is_ascii_stop(u8 c) { return char_is_type(c, (char_type)(CHAR_TYPE_ESC_ASCII | CHAR_TYPE_NON_ASCII)); } -/** Match a line end character: '\\n', '\\r', '\0'*/ +/** Match a line end character: '\\n', '\\r', '\0'. */ static_inline bool char_is_line_end(u8 c) { return char_is_type(c, (char_type)CHAR_TYPE_LINE_END); } +/** Match a hexadecimal numeric character: [0-9a-fA-F]. */ +static_inline bool char_is_hex(u8 c) { + return char_is_type(c, (char_type)CHAR_TYPE_HEX); +} + /*============================================================================== @@ -2716,6 +3679,8 @@ static_inline bool digi_is_digit_or_fp(u8 d) { +#if !YYJSON_DISABLE_READER + /*============================================================================== * Hex Character Reader * This function is used by JSON reader to read escaped characters. @@ -2825,7 +3790,6 @@ static_inline bool read_null(u8 **ptr, yyjson_val *val) { /** Read 'Inf' or 'Infinity' literal (ignoring case). */ static_inline bool read_inf(bool sign, u8 **ptr, u8 **pre, yyjson_val *val) { -#if !YYJSON_DISABLE_NON_STANDARD u8 *hdr = *ptr - sign; u8 *cur = *ptr; u8 **end = ptr; @@ -2842,7 +3806,7 @@ static_inline bool read_inf(bool sign, u8 **ptr, u8 **pre, yyjson_val *val) { cur += 3; } *end = cur; - if (pre) { + if (false) { /* add null-terminator for previous raw string */ if (*pre) **pre = '\0'; *pre = cur; @@ -2854,13 +3818,11 @@ static_inline bool read_inf(bool sign, u8 **ptr, u8 **pre, yyjson_val *val) { } return true; } -#endif return false; } /** Read 'NaN' literal (ignoring case). */ static_inline bool read_nan(bool sign, u8 **ptr, u8 **pre, yyjson_val *val) { -#if !YYJSON_DISABLE_NON_STANDARD u8 *hdr = *ptr - sign; u8 *cur = *ptr; u8 **end = ptr; @@ -2869,7 +3831,7 @@ static_inline bool read_nan(bool sign, u8 **ptr, u8 **pre, yyjson_val *val) { (cur[2] == 'N' || cur[2] == 'n')) { cur += 3; *end = cur; - if (pre) { + if (false) { /* add null-terminator for previous raw string */ if (*pre) **pre = '\0'; *pre = cur; @@ -2881,22 +3843,13 @@ static_inline bool read_nan(bool sign, u8 **ptr, u8 **pre, yyjson_val *val) { } return true; } -#endif - return false; -} - -/** Read 'Inf', 'Infinity' or 'NaN' literal (ignoring case). */ -static_inline bool read_inf_or_nan(bool sign, u8 **ptr, u8 **pre, - yyjson_val *val) { - if (read_inf(sign, ptr, pre, val)) return true; - if (read_nan(sign, ptr, pre, val)) return true; return false; } /** Read a JSON number as raw string. */ static_noinline bool read_number_raw(u8 **ptr, u8 **pre, - bool ext, + yyjson_read_flag flg, yyjson_val *val, const char **msg) { @@ -2924,10 +3877,7 @@ static_noinline bool read_number_raw(u8 **ptr, /* read first digit, check leading zero */ if (unlikely(!digi_is_digit(*cur))) { - if (unlikely(ext)) { - if (read_inf_or_nan(*hdr == '-', &cur, pre, val)) return_raw(); - } - return_err(cur - 1, "no digit after minus sign"); + return_err(cur, "no digit after minus sign"); } /* read integral part */ @@ -2946,7 +3896,7 @@ static_noinline bool read_number_raw(u8 **ptr, if (*cur == '.') { cur++; if (!digi_is_digit(*cur++)) { - return_err(cur - 1, "no digit after decimal point"); + return_err(cur, "no digit after decimal point"); } while (digi_is_digit(*cur)) cur++; } @@ -2955,7 +3905,7 @@ static_noinline bool read_number_raw(u8 **ptr, if (digi_is_exp(*cur)) { cur += 1 + digi_is_sign(cur[1]); if (!digi_is_digit(*cur++)) { - return_err(cur - 1, "no digit after exponent sign"); + return_err(cur, "no digit after exponent sign"); } while (digi_is_digit(*cur)) cur++; } @@ -3011,6 +3961,114 @@ static_noinline bool skip_spaces_and_comments(u8 **ptr) { return hdr != cur; } +/** + Check truncated string. + Returns true if `cur` match `str` but is truncated. + */ +static_inline bool is_truncated_str(u8 *cur, u8 *end, + const char *str, + bool case_sensitive) { + usize len = strlen(str); + if (cur + len <= end || end <= cur) return false; + if (case_sensitive) { + return memcmp(cur, str, (usize)(end - cur)) == 0; + } + for (; cur < end; cur++, str++) { + if ((*cur != (u8)*str) && (*cur != (u8)*str - 'a' + 'A')) { + return false; + } + } + return true; +} + +/** + Check truncated JSON on parsing errors. + Returns true if the input is valid but truncated. + */ +static_noinline bool is_truncated_end(u8 *hdr, u8 *cur, u8 *end, + yyjson_read_code code) { + if (cur >= end) return true; + if (code == YYJSON_READ_ERROR_LITERAL) { + if (is_truncated_str(cur, end, "true", true) || + is_truncated_str(cur, end, "false", true) || + is_truncated_str(cur, end, "null", true)) { + return true; + } + } + if (code == YYJSON_READ_ERROR_UNEXPECTED_CHARACTER || + code == YYJSON_READ_ERROR_INVALID_NUMBER || + code == YYJSON_READ_ERROR_LITERAL) { + if (false) { + if (*cur == '-') cur++; + if (is_truncated_str(cur, end, "infinity", false) || + is_truncated_str(cur, end, "nan", false)) { + return true; + } + } + } + if (code == YYJSON_READ_ERROR_UNEXPECTED_CONTENT) { + if (false) { + if (hdr + 3 <= cur && + is_truncated_str(cur - 3, end, "infinity", false)) { + return true; /* e.g. infin would be read as inf + in */ + } + } + } + if (code == YYJSON_READ_ERROR_INVALID_STRING) { + usize len = (usize)(end - cur); + + /* unicode escape sequence */ + if (*cur == '\\') { + if (len == 1) return true; + if (len <= 5) { + if (*++cur != 'u') return false; + for (++cur; cur < end; cur++) { + if (!char_is_hex(*cur)) return false; + } + return true; + } + return false; + } + + /* 2 to 4 bytes UTF-8, see `read_string()` for details. */ + if (*cur & 0x80) { + u8 c0 = cur[0], c1 = cur[1], c2 = cur[2]; + if (len == 1) { + /* 2 bytes UTF-8, truncated */ + if ((c0 & 0xE0) == 0xC0 && (c0 & 0x1E) != 0x00) return true; + /* 3 bytes UTF-8, truncated */ + if ((c0 & 0xF0) == 0xE0) return true; + /* 4 bytes UTF-8, truncated */ + if ((c0 & 0xF8) == 0xF0 && (c0 & 0x07) <= 0x04) return true; + } + if (len == 2) { + /* 3 bytes UTF-8, truncated */ + if ((c0 & 0xF0) == 0xE0 && + (c1 & 0xC0) == 0x80) { + u8 pat = (u8)(((c0 & 0x0F) << 1) | ((c1 & 0x20) >> 5)); + return 0x01 <= pat && pat != 0x1B; + } + /* 4 bytes UTF-8, truncated */ + if ((c0 & 0xF8) == 0xF0 && + (c1 & 0xC0) == 0x80) { + u8 pat = (u8)(((c0 & 0x07) << 2) | ((c1 & 0x30) >> 4)); + return 0x01 <= pat && pat <= 0x10; + } + } + if (len == 3) { + /* 4 bytes UTF-8, truncated */ + if ((c0 & 0xF8) == 0xF0 && + (c1 & 0xC0) == 0x80 && + (c2 & 0xC0) == 0x80) { + u8 pat = (u8)(((c0 & 0x07) << 2) | ((c1 & 0x30) >> 4)); + return 0x01 <= pat && pat <= 0x10; + } + } + } + } + return false; +} + #if YYJSON_HAS_IEEE_754 && !YYJSON_DISABLE_FAST_FP_CONV /* FP_READER */ @@ -3298,8 +4356,6 @@ static const f64 f64_pow10_table[] = { 3. This function (with inline attribute) may generate a lot of instructions. */ static_inline bool read_number(u8 **ptr, - u8 **pre, - bool ext, yyjson_val *val, const char **msg) { @@ -3309,6 +4365,12 @@ static_inline bool read_number(u8 **ptr, return false; \ } while (false) +#define return_0() do { \ + val->tag = YYJSON_TYPE_NUM | (u8)((u8)sign << 3); \ + val->uni.u64 = 0; \ + *end = cur; return true; \ +} while (false) + #define return_i64(_v) do { \ val->tag = YYJSON_TYPE_NUM | (u8)((u8)sign << 3); \ val->uni.u64 = (u64)(sign ? (u64)(~(_v) + 1) : (u64)(_v)); \ @@ -3321,14 +4383,14 @@ static_inline bool read_number(u8 **ptr, *end = cur; return true; \ } while (false) -#define return_f64_raw(_v) do { \ +#define return_f64_bin(_v) do { \ val->tag = YYJSON_TYPE_NUM | YYJSON_SUBTYPE_REAL; \ val->uni.u64 = ((u64)sign << 63) | (u64)(_v); \ *end = cur; return true; \ } while (false) #define return_inf() do { \ - if (unlikely(ext)) return_f64_raw(F64_RAW_INF); \ + if (false) return_f64_bin(F64_RAW_INF); \ else return_err(hdr, "number is infinity when parsed as double"); \ } while (false) @@ -3350,31 +4412,20 @@ static_inline bool read_number(u8 **ptr, u8 **end = ptr; bool sign; - /* read number as raw string if has flag */ - if (unlikely(pre)) { - return read_number_raw(ptr, pre, ext, val, msg); - } - sign = (*hdr == '-'); cur += sign; /* begin with a leading zero or non-digit */ if (unlikely(!digi_is_nonzero(*cur))) { /* 0 or non-digit char */ if (unlikely(*cur != '0')) { /* non-digit char */ - if (unlikely(ext)) { - if (read_inf_or_nan(sign, &cur, pre, val)) { - *end = cur; - return true; - } - } - return_err(cur - 1, "no digit after minus sign"); + return_err(cur, "no digit after minus sign"); } /* begin with 0 */ - if (likely(!digi_is_digit_or_fp(*++cur))) return_i64(0); + if (likely(!digi_is_digit_or_fp(*++cur))) return_0(); if (likely(*cur == '.')) { dot_pos = cur++; if (unlikely(!digi_is_digit(*cur))) { - return_err(cur - 1, "no digit after decimal point"); + return_err(cur, "no digit after decimal point"); } while (unlikely(*cur == '0')) cur++; if (likely(digi_is_digit(*cur))) { @@ -3390,11 +4441,11 @@ static_inline bool read_number(u8 **ptr, if (unlikely(digi_is_exp(*cur))) { /* 0 with any exponent is still 0 */ cur += (usize)1 + digi_is_sign(cur[1]); if (unlikely(!digi_is_digit(*cur))) { - return_err(cur - 1, "no digit after exponent sign"); + return_err(cur, "no digit after exponent sign"); } while (digi_is_digit(*++cur)); } - return_f64_raw(0); + return_f64_bin(0); } /* begin with non-zero digit */ @@ -3412,7 +4463,7 @@ static_inline bool read_number(u8 **ptr, #define expr_intg(i) \ if (likely((num = (u64)(cur[i] - (u8)'0')) <= 9)) sig = num + sig * 10; \ else { goto digi_sepr_##i; } - repeat_in_1_18(expr_intg); + repeat_in_1_18(expr_intg) #undef expr_intg @@ -3472,7 +4523,9 @@ static_inline bool read_number(u8 **ptr, sig = num + sig * 10; cur++; /* convert to double if overflow */ - if (sign) return_f64(normalized_u64_to_f64(sig)); + if (sign) { + return_f64(normalized_u64_to_f64(sig)); + } return_i64(sig); } } @@ -3524,13 +4577,13 @@ static_inline bool read_number(u8 **ptr, /* fraction part end */ digi_frac_end: if (unlikely(dot_pos + 1 == cur)) { - return_err(cur - 1, "no digit after decimal point"); + return_err(cur, "no digit after decimal point"); } sig_end = cur; exp_sig = -(i64)((u64)(cur - dot_pos) - 1); if (likely(!digi_is_exp(*cur))) { if (unlikely(exp_sig < F64_MIN_DEC_EXP - 19)) { - return_f64_raw(0); /* underflow */ + return_f64_bin(0); /* underflow */ } exp = (i32)exp_sig; goto digi_finish; @@ -3544,7 +4597,7 @@ static_inline bool read_number(u8 **ptr, exp_sign = (*++cur == '-'); cur += digi_is_sign(*cur); if (unlikely(!digi_is_digit(*cur))) { - return_err(cur - 1, "no digit after exponent sign"); + return_err(cur, "no digit after exponent sign"); } while (*cur == '0') cur++; @@ -3555,7 +4608,7 @@ static_inline bool read_number(u8 **ptr, } if (unlikely(cur - tmp >= U64_SAFE_DIG)) { if (exp_sign) { - return_f64_raw(0); /* underflow */ + return_f64_bin(0); /* underflow */ } else { return_inf(); /* overflow */ } @@ -3566,7 +4619,7 @@ static_inline bool read_number(u8 **ptr, /* validate exponent value */ digi_exp_finish: if (unlikely(exp_sig < F64_MIN_DEC_EXP - 19)) { - return_f64_raw(0); /* underflow */ + return_f64_bin(0); /* underflow */ } if (unlikely(exp_sig > F64_MAX_DEC_EXP)) { return_inf(); /* overflow */ @@ -3737,7 +4790,7 @@ static_inline bool read_number(u8 **ptr, exp2 += F64_BITS - F64_SIG_FULL_BITS + F64_SIG_BITS; exp2 += F64_EXP_BIAS; raw = ((u64)exp2 << F64_SIG_BITS) | (hi & F64_SIG_MASK); - return_f64_raw(raw); + return_f64_bin(raw); } } @@ -3827,7 +4880,7 @@ static_inline bool read_number(u8 **ptr, if (unlikely(raw == F64_RAW_INF)) return_inf(); if (likely(precision_bits <= half_way - fp_err || precision_bits >= half_way + fp_err)) { - return_f64_raw(raw); /* number is accurate */ + return_f64_bin(raw); /* number is accurate */ } /* now the number is the correct value, or the next lower value */ @@ -3867,15 +4920,16 @@ static_inline bool read_number(u8 **ptr, } if (unlikely(raw == F64_RAW_INF)) return_inf(); - return_f64_raw(raw); + return_f64_bin(raw); } -#undef has_flag #undef return_err #undef return_inf +#undef return_0 #undef return_i64 #undef return_f64 -#undef return_f64_raw +#undef return_f64_bin +#undef return_raw } @@ -3887,11 +4941,9 @@ static_inline bool read_number(u8 **ptr, This is a fallback function if the custom number reader is disabled. This function use libc's strtod() to read floating-point number. */ -static_noinline bool read_number(u8 **ptr, - u8 **pre, - bool ext, - yyjson_val *val, - const char **msg) { +static_inline bool read_number(u8 **ptr, + yyjson_val *val, + const char **msg) { #define return_err(_pos, _msg) do { \ *msg = _msg; \ @@ -3899,6 +4951,12 @@ static_noinline bool read_number(u8 **ptr, return false; \ } while (false) +#define return_0() do { \ + val->tag = YYJSON_TYPE_NUM | (u64)((u8)sign << 3); \ + val->uni.u64 = 0; \ + *end = cur; return true; \ +} while (false) + #define return_i64(_v) do { \ val->tag = YYJSON_TYPE_NUM | (u64)((u8)sign << 3); \ val->uni.u64 = (u64)(sign ? (u64)(~(_v) + 1) : (u64)(_v)); \ @@ -3911,12 +4969,17 @@ static_noinline bool read_number(u8 **ptr, *end = cur; return true; \ } while (false) -#define return_f64_raw(_v) do { \ +#define return_f64_bin(_v) do { \ val->tag = YYJSON_TYPE_NUM | YYJSON_SUBTYPE_REAL; \ val->uni.u64 = ((u64)sign << 63) | (u64)(_v); \ *end = cur; return true; \ } while (false) +#define return_inf() do { \ + if (false) return_f64_bin(F64_RAW_INF); \ + else return_err(hdr, "number is infinity when parsed as double"); \ +} while (false) + u64 sig, num; u8 *hdr = *ptr; u8 *cur = *ptr; @@ -3925,9 +4988,9 @@ static_noinline bool read_number(u8 **ptr, u8 *f64_end = NULL; bool sign; - /* read number as raw string if has flag */ - if (unlikely(pre)) { - return read_number_raw(ptr, pre, ext, val, msg); + /* read number as raw string if has `YYJSON_READ_NUMBER_AS_RAW` flag */ + if (unlikely(false)) { + return read_number_raw(ptr, pre, flg, val, msg); } sign = (*hdr == '-'); @@ -3936,20 +4999,14 @@ static_noinline bool read_number(u8 **ptr, /* read first digit, check leading zero */ if (unlikely(!digi_is_digit(*cur))) { - if (unlikely(ext)) { - if (read_inf_or_nan(sign, &cur, pre, val)) { - *end = cur; - return true; - } - } - return_err(cur - 1, "no digit after minus sign"); + return_err(cur, "no digit after minus sign"); } if (*cur == '0') { cur++; if (unlikely(digi_is_digit(*cur))) { return_err(cur - 1, "number with leading zero is not allowed"); } - if (!digi_is_fp(*cur)) return_i64(0); + if (!digi_is_fp(*cur)) return_0(); goto read_double; } @@ -3957,7 +5014,7 @@ static_noinline bool read_number(u8 **ptr, #define expr_intg(i) \ if (likely((num = (u64)(cur[i] - (u8)'0')) <= 9)) sig = num + sig * 10; \ else { cur += i; goto intg_end; } - repeat_in_1_18(expr_intg); + repeat_in_1_18(expr_intg) #undef expr_intg /* here are 19 continuous digits, skip them */ @@ -3969,7 +5026,10 @@ static_noinline bool read_number(u8 **ptr, (sig == (U64_MAX / 10) && num <= (U64_MAX % 10))) { sig = num + sig * 10; cur++; - if (sign) return_f64(normalized_u64_to_f64(sig)); + if (sign) { + if (false) return_raw(); + return_f64(normalized_u64_to_f64(sig)); + } return_i64(sig); } } @@ -3979,6 +5039,7 @@ static_noinline bool read_number(u8 **ptr, if (!digi_is_digit_or_fp(*cur)) { /* this number is an integer consisting of 1 to 19 digits */ if (sign && (sig > ((u64)1 << 63))) { + if (false) return_raw(); return_f64(normalized_u64_to_f64(sig)); } return_i64(sig); @@ -3991,17 +5052,19 @@ static_noinline bool read_number(u8 **ptr, /* skip fraction part */ dot = cur; cur++; - if (!digi_is_digit(*cur++)) { - return_err(cur - 1, "no digit after decimal point"); + if (!digi_is_digit(*cur)) { + return_err(cur, "no digit after decimal point"); } + cur++; while (digi_is_digit(*cur)) cur++; } if (digi_is_exp(*cur)) { /* skip exponent part */ cur += 1 + digi_is_sign(cur[1]); - if (!digi_is_digit(*cur++)) { - return_err(cur - 1, "no digit after exponent sign"); + if (!digi_is_digit(*cur)) { + return_err(cur, "no digit after exponent sign"); } + cur++; while (digi_is_digit(*cur)) cur++; } @@ -4019,28 +5082,32 @@ static_noinline bool read_number(u8 **ptr, */ val->uni.f64 = strtod((const char *)hdr, (char **)&f64_end); if (unlikely(f64_end != cur)) { + /* replace '.' with ',' for locale */ bool cut = (*cur == ','); - if (dot) *dot = ','; if (cut) *cur = ' '; + if (dot) *dot = ','; val->uni.f64 = strtod((const char *)hdr, (char **)&f64_end); + /* restore ',' to '.' */ if (cut) *cur = ','; + if (dot) *dot = '.'; if (unlikely(f64_end != cur)) { return_err(hdr, "strtod() failed to parse the number"); } } - if (unlikely(val->uni.f64 == HUGE_VAL || val->uni.f64 == -HUGE_VAL)) { - if (!ext) { - return_err(hdr, "number is infinity when parsed as double"); - } + if (unlikely(val->uni.f64 >= HUGE_VAL || val->uni.f64 <= -HUGE_VAL)) { + return_inf(); } val->tag = YYJSON_TYPE_NUM | YYJSON_SUBTYPE_REAL; *end = cur; return true; -#undef has_flag #undef return_err +#undef return_0 #undef return_i64 #undef return_f64 +#undef return_f64_bin +#undef return_inf +#undef return_raw } #endif /* FP_READER */ @@ -4062,7 +5129,6 @@ static_noinline bool read_number(u8 **ptr, */ static_inline bool read_string(u8 **ptr, u8 *lst, - bool inv, yyjson_val *val, const char **msg) { /* @@ -4139,6 +5205,7 @@ static_inline bool read_string(u8 **ptr, const u32 b4_err0 = 0x00000004UL; const u32 b4_err1 = 0x00003003UL; #else + /* this should be evaluated at compile-time */ v32_uni b1_mask_uni = {{ 0x80, 0x00, 0x00, 0x00 }}; v32_uni b1_patt_uni = {{ 0x00, 0x00, 0x00, 0x00 }}; v32_uni b2_mask_uni = {{ 0xE0, 0xC0, 0x00, 0x00 }}; @@ -4198,7 +5265,7 @@ static_inline bool read_string(u8 **ptr, u8 *cur = *ptr; u8 **end = ptr; - u8 *src = ++cur, *dst, *pos; + u8 *src = ++cur, *dst; u16 hi, lo; u32 uni, tmp; @@ -4215,7 +5282,7 @@ static_inline bool read_string(u8 **ptr, while (true) repeat16({ if (likely(!(char_is_ascii_stop(*src)))) src++; else break; - }); + }) */ #define expr_jump(i) \ if (likely(!char_is_ascii_stop(src[i]))) {} \ @@ -4226,10 +5293,10 @@ static_inline bool read_string(u8 **ptr, src += i; \ goto skip_ascii_end; - repeat16_incr(expr_jump); + repeat16_incr(expr_jump) src += 16; goto skip_ascii_begin; - repeat16_incr(expr_stop); + repeat16_incr(expr_stop) #undef expr_jump #undef expr_stop @@ -4245,10 +5312,11 @@ static_inline bool read_string(u8 **ptr, MSVC, Clang, ICC can generate expected instructions without this hint. */ #if YYJSON_IS_REAL_GCC - __asm volatile("":"=m"(*src)::); + __asm__ volatile("":"=m"(*src)); #endif if (likely(*src == '"')) { - val->tag = ((u64)(src - cur) << YYJSON_TAG_BIT) | YYJSON_TYPE_STR; + val->tag = ((u64)(src - cur) << YYJSON_TAG_BIT) | + (u64)(YYJSON_TYPE_STR); val->uni.str = (const char *)cur; *src = '\0'; *end = src + 1; @@ -4264,7 +5332,21 @@ static_inline bool read_string(u8 **ptr, consecutively. We process the byte sequences of the same length in each loop, which is more friendly to branch prediction. */ - pos = src; +#if YYJSON_DISABLE_UTF8_VALIDATION + while (true) repeat8({ + if (likely((*src & 0xF0) == 0xE0)) src += 3; + else break; + }) + if (*src < 0x80) goto skip_ascii; + while (true) repeat8({ + if (likely((*src & 0xE0) == 0xC0)) src += 2; + else break; + }) + while (true) repeat8({ + if (likely((*src & 0xF8) == 0xF0)) src += 4; + else break; + }) +#else uni = byte_load_4(src); while (is_valid_seq_3(uni)) { src += 3; @@ -4279,10 +5361,7 @@ static_inline bool read_string(u8 **ptr, src += 4; uni = byte_load_4(src); } - if (unlikely(pos == src)) { - if (!inv) return_err(src, "invalid UTF-8 encoding in string"); - ++src; - } +#endif goto skip_ascii; } @@ -4301,7 +5380,7 @@ static_inline bool read_string(u8 **ptr, case 't': *dst++ = '\t'; src++; break; case 'u': if (unlikely(!read_hex_u16(++src, &hi))) { - return_err(src - 2, "invalid escaped unicode in string"); + return_err(src - 2, "invalid escaped sequence in string"); } src += 4; if (likely((hi & 0xF800) != 0xD800)) { @@ -4321,9 +5400,11 @@ static_inline bool read_string(u8 **ptr, if (unlikely((hi & 0xFC00) != 0xD800)) { return_err(src - 6, "invalid high surrogate in string"); } - if (unlikely(!byte_match_2(src, "\\u")) || - unlikely(!read_hex_u16(src + 2, &lo))) { - return_err(src, "no matched low surrogate in string"); + if (unlikely(!byte_match_2(src, "\\u"))) { + return_err(src, "no low surrogate in string"); + } + if (unlikely(!read_hex_u16(src + 2, &lo))) { + return_err(src, "invalid escaped sequence in string"); } if (unlikely((lo & 0xFC00) != 0xDC00)) { return_err(src, "invalid low surrogate in string"); @@ -4342,13 +5423,10 @@ static_inline bool read_string(u8 **ptr, } else if (likely(*src == '"')) { val->tag = ((u64)(dst - cur) << YYJSON_TAG_BIT) | YYJSON_TYPE_STR; val->uni.str = (const char *)cur; - *dst = '\0'; *end = src + 1; return true; } else { - if (!inv) return_err(src, "unexpected control character in string"); - if (src >= lst) return_err(src, "unclosed string"); - *dst++ = *src++; + return_err(src, "unexpected control character in string"); } copy_ascii: @@ -4358,18 +5436,18 @@ static_inline bool read_string(u8 **ptr, while (true) repeat16({ if (unlikely(char_is_ascii_stop(*src))) break; *dst++ = *src++; - }); + }) */ #if YYJSON_IS_REAL_GCC # define expr_jump(i) \ if (likely(!(char_is_ascii_stop(src[i])))) {} \ - else { __asm volatile("":"=m"(src[i])::); goto copy_ascii_stop_##i; } + else { __asm__ volatile("":"=m"(src[i])); goto copy_ascii_stop_##i; } #else # define expr_jump(i) \ if (likely(!(char_is_ascii_stop(src[i])))) {} \ else { goto copy_ascii_stop_##i; } #endif - repeat16_incr(expr_jump); + repeat16_incr(expr_jump) #undef expr_jump byte_move_16(dst, src); @@ -4377,6 +5455,10 @@ static_inline bool read_string(u8 **ptr, dst += 16; goto copy_ascii; + /* + The memory will be moved forward by at least 1 byte. So the `byte_move` + can be one byte more than needed to reduce the number of instructions. + */ copy_ascii_stop_0: goto copy_utf8; copy_ascii_stop_1: @@ -4467,31 +5549,54 @@ static_inline bool read_string(u8 **ptr, copy_utf8: if (*src & 0x80) { /* non-ASCII character */ - pos = src; uni = byte_load_4(src); +#if YYJSON_DISABLE_UTF8_VALIDATION + while (true) repeat4({ + if ((uni & b3_mask) == b3_patt) { + byte_copy_4(dst, &uni); + dst += 3; + src += 3; + uni = byte_load_4(src); + } else break; + }) + if ((uni & b1_mask) == b1_patt) goto copy_ascii; + while (true) repeat4({ + if ((uni & b2_mask) == b2_patt) { + byte_copy_2(dst, &uni); + dst += 2; + src += 2; + uni = byte_load_4(src); + } else break; + }) + while (true) repeat4({ + if ((uni & b4_mask) == b4_patt) { + byte_copy_4(dst, &uni); + dst += 4; + src += 4; + uni = byte_load_4(src); + } else break; + }) +#else while (is_valid_seq_3(uni)) { - byte_move_4(dst, &uni); + byte_copy_4(dst, &uni); dst += 3; src += 3; uni = byte_load_4(src); } if (is_valid_seq_1(uni)) goto copy_ascii; while (is_valid_seq_2(uni)) { - byte_move_2(dst, &uni); + byte_copy_2(dst, &uni); dst += 2; src += 2; uni = byte_load_4(src); } while (is_valid_seq_4(uni)) { - byte_move_4(dst, &uni); + byte_copy_4(dst, &uni); dst += 4; src += 4; uni = byte_load_4(src); } - if (unlikely(pos == src)) { - if (!inv) return_err(src, "invalid UTF-8 encoding in string"); - goto copy_ascii_stop_1; - } +#endif goto copy_ascii; } goto copy_escape; @@ -4518,13 +5623,10 @@ static_noinline yyjson_doc *read_root_single(u8 *hdr, u8 *cur, u8 *end, yyjson_alc alc, - yyjson_read_flag flg, yyjson_read_err *err) { -#define has_flag(_flag) unlikely((flg & YYJSON_READ_##_flag) != 0) - #define return_err(_pos, _code, _msg) do { \ - if (_pos >= end) { \ + if (is_truncated_end(hdr, _pos, end, YYJSON_READ_ERROR_##_code)) { \ err->pos = (usize)(end - hdr); \ err->code = YYJSON_READ_ERROR_UNEXPECTED_END; \ err->msg = "unexpected end of data"; \ @@ -4544,12 +5646,6 @@ static_noinline yyjson_doc *read_root_single(u8 *hdr, yyjson_doc *doc; /* the JSON document, equals to val_hdr */ const char *msg; /* error message */ - bool raw; /* read number as raw */ - bool ext; /* allow inf and nan */ - bool inv; /* allow invalid unicode */ - u8 *raw_end; /* raw end for null-terminator */ - u8 **pre; /* previous raw end pointer */ - hdr_len = sizeof(yyjson_doc) / sizeof(yyjson_val); hdr_len += (sizeof(yyjson_doc) % sizeof(yyjson_val)) > 0; alc_num = hdr_len + 1; /* single value */ @@ -4557,18 +5653,13 @@ static_noinline yyjson_doc *read_root_single(u8 *hdr, val_hdr = (yyjson_val *)alc.malloc(alc.ctx, alc_num * sizeof(yyjson_val)); if (unlikely(!val_hdr)) goto fail_alloc; val = val_hdr + hdr_len; - raw = (flg & YYJSON_READ_NUMBER_AS_RAW) != 0; - ext = (flg & YYJSON_READ_ALLOW_INF_AND_NAN) != 0; - inv = (flg & YYJSON_READ_ALLOW_INVALID_UNICODE) != 0; - raw_end = NULL; - pre = raw ? &raw_end : NULL; if (char_is_number(*cur)) { - if (likely(read_number(&cur, pre, ext, val, &msg))) goto doc_end; + if (likely(read_number(&cur, val, &msg))) goto doc_end; goto fail_number; } if (*cur == '"') { - if (likely(read_string(&cur, end, inv, val, &msg))) goto doc_end; + if (likely(read_string(&cur, end, val, &msg))) goto doc_end; goto fail_string; } if (*cur == 't') { @@ -4581,20 +5672,17 @@ static_noinline yyjson_doc *read_root_single(u8 *hdr, } if (*cur == 'n') { if (likely(read_null(&cur, val))) goto doc_end; - if (unlikely(ext)) { - if (read_nan(false, &cur, pre, val)) goto doc_end; + if (false) { + if (read_nan(false, &cur, 0, val)) goto doc_end; } goto fail_literal; } - if (unlikely(ext)) { - if (read_inf_or_nan(false, &cur, pre, val)) goto doc_end; - } goto fail_character; doc_end: /* check invalid contents after json document */ - if (unlikely(cur < end) && !has_flag(STOP_WHEN_DONE)) { - if (has_flag(ALLOW_COMMENTS)) { + if (unlikely(cur < end) && !has_read_flag(STOP_WHEN_DONE)) { + if (false) { if (!skip_spaces_and_comments(&cur)) { if (byte_match_2(cur, "/*")) goto fail_comment; } @@ -4604,13 +5692,12 @@ static_noinline yyjson_doc *read_root_single(u8 *hdr, if (unlikely(cur < end)) goto fail_garbage; } - if (pre && *pre) **pre = '\0'; doc = (yyjson_doc *)val_hdr; doc->root = val_hdr + hdr_len; doc->alc = alc; doc->dat_read = (usize)(cur - hdr); doc->val_read = 1; - doc->str_pool = has_flag(INSITU) ? NULL : (char *)hdr; + doc->str_pool = (char *)hdr; return doc; fail_string: @@ -4627,8 +5714,9 @@ static_noinline yyjson_doc *read_root_single(u8 *hdr, return_err(cur, UNEXPECTED_CHARACTER, "unexpected character"); fail_garbage: return_err(cur, UNEXPECTED_CONTENT, "unexpected content after document"); +fail_recursion: + return_err(cur, RECURSION_DEPTH, "array and object recursion depth exceeded"); -#undef has_flag #undef return_err } @@ -4637,13 +5725,10 @@ static_inline yyjson_doc *read_root_minify(u8 *hdr, u8 *cur, u8 *end, yyjson_alc alc, - yyjson_read_flag flg, yyjson_read_err *err) { -#define has_flag(_flag) unlikely((flg & YYJSON_READ_##_flag) != 0) - #define return_err(_pos, _code, _msg) do { \ - if (_pos >= end) { \ + if (is_truncated_end(hdr, _pos, end, YYJSON_READ_ERROR_##_code)) { \ err->pos = (usize)(end - hdr); \ err->code = YYJSON_READ_ERROR_UNEXPECTED_END; \ err->msg = "unexpected end of data"; \ @@ -4659,9 +5744,11 @@ static_inline yyjson_doc *read_root_minify(u8 *hdr, #define val_incr() do { \ val++; \ if (unlikely(val >= val_end)) { \ + usize alc_old = alc_len; \ alc_len += alc_len / 2; \ - if ((alc_len >= alc_max)) goto fail_alloc; \ + if ((sizeof(usize) < 8) && (alc_len >= alc_max)) goto fail_alloc; \ val_tmp = (yyjson_val *)alc.realloc(alc.ctx, (void *)val_hdr, \ + alc_old * sizeof(yyjson_val), \ alc_len * sizeof(yyjson_val)); \ if ((!val_tmp)) goto fail_alloc; \ val = val_tmp + (usize)(val - val_hdr); \ @@ -4684,14 +5771,12 @@ static_inline yyjson_doc *read_root_minify(u8 *hdr, yyjson_val *ctn_parent; /* parent of current container */ yyjson_doc *doc; /* the JSON document, equals to val_hdr */ const char *msg; /* error message */ - + + u32 container_depth = 0; /* limit on number of open array and map */ bool raw; /* read number as raw */ - bool ext; /* allow inf and nan */ bool inv; /* allow invalid unicode */ - u8 *raw_end; /* raw end for null-terminator */ - u8 **pre; /* previous raw end pointer */ - dat_len = has_flag(STOP_WHEN_DONE) ? 256 : (usize)(end - cur); + dat_len = has_read_flag(STOP_WHEN_DONE) ? 256 : (usize)(end - cur); hdr_len = sizeof(yyjson_doc) / sizeof(yyjson_val); hdr_len += (sizeof(yyjson_doc) % sizeof(yyjson_val)) > 0; alc_max = USIZE_MAX / sizeof(yyjson_val); @@ -4704,12 +5789,7 @@ static_inline yyjson_doc *read_root_minify(u8 *hdr, val = val_hdr + hdr_len; ctn = val; ctn_len = 0; - raw = (flg & YYJSON_READ_NUMBER_AS_RAW) != 0; - ext = (flg & YYJSON_READ_ALLOW_INF_AND_NAN) != 0; - inv = (flg & YYJSON_READ_ALLOW_INVALID_UNICODE) != 0; - raw_end = NULL; - pre = raw ? &raw_end : NULL; - + if (*cur++ == '{') { ctn->tag = YYJSON_TYPE_OBJ; ctn->uni.ofs = 0; @@ -4721,6 +5801,11 @@ static_inline yyjson_doc *read_root_minify(u8 *hdr, } arr_begin: + container_depth++; + if (unlikely(container_depth >= YYJSON_READER_CONTAINER_RECURSION_LIMIT)) { + goto fail_recursion; + } + /* save current container */ ctn->tag = (((u64)ctn_len + 1) << YYJSON_TAG_BIT) | (ctn->tag & YYJSON_TAG_MASK); @@ -4746,13 +5831,13 @@ static_inline yyjson_doc *read_root_minify(u8 *hdr, if (char_is_number(*cur)) { val_incr(); ctn_len++; - if (likely(read_number(&cur, pre, ext, val, &msg))) goto arr_val_end; + if (likely(read_number(&cur, val, &msg))) goto arr_val_end; goto fail_number; } if (*cur == '"') { val_incr(); ctn_len++; - if (likely(read_string(&cur, end, inv, val, &msg))) goto arr_val_end; + if (likely(read_string(&cur, end, val, &msg))) goto arr_val_end; goto fail_string; } if (*cur == 't') { @@ -4771,31 +5856,18 @@ static_inline yyjson_doc *read_root_minify(u8 *hdr, val_incr(); ctn_len++; if (likely(read_null(&cur, val))) goto arr_val_end; - if (unlikely(ext)) { - if (read_nan(false, &cur, pre, val)) goto arr_val_end; - } goto fail_literal; } if (*cur == ']') { cur++; if (likely(ctn_len == 0)) goto arr_end; - if (has_flag(ALLOW_TRAILING_COMMAS)) goto arr_end; + while (*cur != ',') cur--; goto fail_trailing_comma; } if (char_is_space(*cur)) { while (char_is_space(*++cur)); goto arr_val_begin; } - if (unlikely(ext) && (*cur == 'i' || *cur == 'I' || *cur == 'N')) { - val_incr(); - ctn_len++; - if (read_inf_or_nan(false, &cur, pre, val)) goto arr_val_end; - goto fail_character; - } - if (has_flag(ALLOW_COMMENTS)) { - if (skip_spaces_and_comments(&cur)) goto arr_val_begin; - if (byte_match_2(cur, "/*")) goto fail_comment; - } goto fail_character; arr_val_end: @@ -4811,13 +5883,15 @@ static_inline yyjson_doc *read_root_minify(u8 *hdr, while (char_is_space(*++cur)); goto arr_val_end; } - if (has_flag(ALLOW_COMMENTS)) { + if (false) { if (skip_spaces_and_comments(&cur)) goto arr_val_end; if (byte_match_2(cur, "/*")) goto fail_comment; } goto fail_character; arr_end: + container_depth--; + /* get parent container */ ctn_parent = (yyjson_val *)(void *)((u8 *)ctn - ctn->uni.ofs); @@ -4836,6 +5910,11 @@ static_inline yyjson_doc *read_root_minify(u8 *hdr, } obj_begin: + container_depth++; + if (unlikely(container_depth >= YYJSON_READER_CONTAINER_RECURSION_LIMIT)) { + goto fail_recursion; + } + /* push container */ ctn->tag = (((u64)ctn_len + 1) << YYJSON_TAG_BIT) | (ctn->tag & YYJSON_TAG_MASK); @@ -4850,20 +5929,20 @@ static_inline yyjson_doc *read_root_minify(u8 *hdr, if (likely(*cur == '"')) { val_incr(); ctn_len++; - if (likely(read_string(&cur, end, inv, val, &msg))) goto obj_key_end; + if (likely(read_string(&cur, end, val, &msg))) goto obj_key_end; goto fail_string; } if (likely(*cur == '}')) { cur++; if (likely(ctn_len == 0)) goto obj_end; - if (has_flag(ALLOW_TRAILING_COMMAS)) goto obj_end; + while (*cur != ',') cur--; goto fail_trailing_comma; } if (char_is_space(*cur)) { while (char_is_space(*++cur)); goto obj_key_begin; } - if (has_flag(ALLOW_COMMENTS)) { + if (false) { if (skip_spaces_and_comments(&cur)) goto obj_key_begin; if (byte_match_2(cur, "/*")) goto fail_comment; } @@ -4878,7 +5957,7 @@ static_inline yyjson_doc *read_root_minify(u8 *hdr, while (char_is_space(*++cur)); goto obj_key_end; } - if (has_flag(ALLOW_COMMENTS)) { + if (false) { if (skip_spaces_and_comments(&cur)) goto obj_key_end; if (byte_match_2(cur, "/*")) goto fail_comment; } @@ -4888,13 +5967,13 @@ static_inline yyjson_doc *read_root_minify(u8 *hdr, if (*cur == '"') { val++; ctn_len++; - if (likely(read_string(&cur, end, inv, val, &msg))) goto obj_val_end; + if (likely(read_string(&cur, end, val, &msg))) goto obj_val_end; goto fail_string; } if (char_is_number(*cur)) { val++; ctn_len++; - if (likely(read_number(&cur, pre, ext, val, &msg))) goto obj_val_end; + if (likely(read_number(&cur, val, &msg))) goto obj_val_end; goto fail_number; } if (*cur == '{') { @@ -4921,25 +6000,12 @@ static_inline yyjson_doc *read_root_minify(u8 *hdr, val++; ctn_len++; if (likely(read_null(&cur, val))) goto obj_val_end; - if (unlikely(ext)) { - if (read_nan(false, &cur, pre, val)) goto obj_val_end; - } goto fail_literal; } if (char_is_space(*cur)) { while (char_is_space(*++cur)); goto obj_val_begin; } - if (unlikely(ext) && (*cur == 'i' || *cur == 'I' || *cur == 'N')) { - val++; - ctn_len++; - if (read_inf_or_nan(false, &cur, pre, val)) goto obj_val_end; - goto fail_character; - } - if (has_flag(ALLOW_COMMENTS)) { - if (skip_spaces_and_comments(&cur)) goto obj_val_begin; - if (byte_match_2(cur, "/*")) goto fail_comment; - } goto fail_character; obj_val_end: @@ -4955,13 +6021,15 @@ static_inline yyjson_doc *read_root_minify(u8 *hdr, while (char_is_space(*++cur)); goto obj_val_end; } - if (has_flag(ALLOW_COMMENTS)) { + if (false) { if (skip_spaces_and_comments(&cur)) goto obj_val_end; if (byte_match_2(cur, "/*")) goto fail_comment; } goto fail_character; obj_end: + container_depth--; + /* pop container */ ctn_parent = (yyjson_val *)(void *)((u8 *)ctn - ctn->uni.ofs); /* point to the next value */ @@ -4978,19 +6046,22 @@ static_inline yyjson_doc *read_root_minify(u8 *hdr, doc_end: /* check invalid contents after json document */ - if (unlikely(cur < end) && !has_flag(STOP_WHEN_DONE)) { - if (has_flag(ALLOW_COMMENTS)) skip_spaces_and_comments(&cur); - else while (char_is_space(*cur)) cur++; + if (unlikely(cur < end) && !has_read_flag(STOP_WHEN_DONE)) { + if (false) { + skip_spaces_and_comments(&cur); + if (byte_match_2(cur, "/*")) goto fail_comment; + } else { + while (char_is_space(*cur)) cur++; + } if (unlikely(cur < end)) goto fail_garbage; } - if (pre && *pre) **pre = '\0'; doc = (yyjson_doc *)val_hdr; doc->root = val_hdr + hdr_len; doc->alc = alc; doc->dat_read = (usize)(cur - hdr); doc->val_read = (usize)((val - doc->root) + 1); - doc->str_pool = has_flag(INSITU) ? NULL : (char *)hdr; + doc->str_pool = has_read_flag(INSITU) ? NULL : (char *)hdr; return doc; fail_string: @@ -5009,8 +6080,9 @@ static_inline yyjson_doc *read_root_minify(u8 *hdr, return_err(cur, UNEXPECTED_CHARACTER, "unexpected character"); fail_garbage: return_err(cur, UNEXPECTED_CONTENT, "unexpected content after document"); +fail_recursion: + return_err(cur, RECURSION_DEPTH, "array and object recursion depth exceeded"); -#undef has_flag #undef val_incr #undef return_err } @@ -5020,13 +6092,10 @@ static_inline yyjson_doc *read_root_pretty(u8 *hdr, u8 *cur, u8 *end, yyjson_alc alc, - yyjson_read_flag flg, yyjson_read_err *err) { -#define has_flag(_flag) unlikely((flg & YYJSON_READ_##_flag) != 0) - #define return_err(_pos, _code, _msg) do { \ - if (_pos >= end) { \ + if (is_truncated_end(hdr, _pos, end, YYJSON_READ_ERROR_##_code)) { \ err->pos = (usize)(end - hdr); \ err->code = YYJSON_READ_ERROR_UNEXPECTED_END; \ err->msg = "unexpected end of data"; \ @@ -5042,9 +6111,11 @@ static_inline yyjson_doc *read_root_pretty(u8 *hdr, #define val_incr() do { \ val++; \ if (unlikely(val >= val_end)) { \ + usize alc_old = alc_len; \ alc_len += alc_len / 2; \ - if ((alc_len >= alc_max)) goto fail_alloc; \ + if ((sizeof(usize) < 8) && (alc_len >= alc_max)) goto fail_alloc; \ val_tmp = (yyjson_val *)alc.realloc(alc.ctx, (void *)val_hdr, \ + alc_old * sizeof(yyjson_val), \ alc_len * sizeof(yyjson_val)); \ if ((!val_tmp)) goto fail_alloc; \ val = val_tmp + (usize)(val - val_hdr); \ @@ -5067,14 +6138,10 @@ static_inline yyjson_doc *read_root_pretty(u8 *hdr, yyjson_val *ctn_parent; /* parent of current container */ yyjson_doc *doc; /* the JSON document, equals to val_hdr */ const char *msg; /* error message */ + + u32 container_depth = 0; /* limit on number of open array and map */ - bool raw; /* read number as raw */ - bool ext; /* allow inf and nan */ - bool inv; /* allow invalid unicode */ - u8 *raw_end; /* raw end for null-terminator */ - u8 **pre; /* previous raw end pointer */ - - dat_len = has_flag(STOP_WHEN_DONE) ? 256 : (usize)(end - cur); + dat_len = has_read_flag(STOP_WHEN_DONE) ? 256 : (usize)(end - cur); hdr_len = sizeof(yyjson_doc) / sizeof(yyjson_val); hdr_len += (sizeof(yyjson_doc) % sizeof(yyjson_val)) > 0; alc_max = USIZE_MAX / sizeof(yyjson_val); @@ -5087,11 +6154,6 @@ static_inline yyjson_doc *read_root_pretty(u8 *hdr, val = val_hdr + hdr_len; ctn = val; ctn_len = 0; - raw = (flg & YYJSON_READ_NUMBER_AS_RAW) != 0; - ext = (flg & YYJSON_READ_ALLOW_INF_AND_NAN) != 0; - inv = (flg & YYJSON_READ_ALLOW_INVALID_UNICODE) != 0; - raw_end = NULL; - pre = raw ? &raw_end : NULL; if (*cur++ == '{') { ctn->tag = YYJSON_TYPE_OBJ; @@ -5106,6 +6168,11 @@ static_inline yyjson_doc *read_root_pretty(u8 *hdr, } arr_begin: + container_depth++; + if (unlikely(container_depth >= YYJSON_READER_CONTAINER_RECURSION_LIMIT)) { + goto fail_recursion; + } + /* save current container */ ctn->tag = (((u64)ctn_len + 1) << YYJSON_TAG_BIT) | (ctn->tag & YYJSON_TAG_MASK); @@ -5125,12 +6192,12 @@ static_inline yyjson_doc *read_root_pretty(u8 *hdr, while (true) repeat16({ if (byte_match_2(cur, " ")) cur += 2; else break; - }); + }) #else while (true) repeat16({ if (likely(byte_match_2(cur, " "))) cur += 2; else break; - }); + }) #endif if (*cur == '{') { @@ -5144,13 +6211,13 @@ static_inline yyjson_doc *read_root_pretty(u8 *hdr, if (char_is_number(*cur)) { val_incr(); ctn_len++; - if (likely(read_number(&cur, pre, ext, val, &msg))) goto arr_val_end; + if (likely(read_number(&cur, val, &msg))) goto arr_val_end; goto fail_number; } if (*cur == '"') { val_incr(); ctn_len++; - if (likely(read_string(&cur, end, inv, val, &msg))) goto arr_val_end; + if (likely(read_string(&cur, end, val, &msg))) goto arr_val_end; goto fail_string; } if (*cur == 't') { @@ -5169,31 +6236,21 @@ static_inline yyjson_doc *read_root_pretty(u8 *hdr, val_incr(); ctn_len++; if (likely(read_null(&cur, val))) goto arr_val_end; - if (unlikely(ext)) { - if (read_nan(false, &cur, pre, val)) goto arr_val_end; + if (false) { + if (read_nan(false, &cur, 0, val)) goto arr_val_end; } goto fail_literal; } if (*cur == ']') { cur++; if (likely(ctn_len == 0)) goto arr_end; - if (has_flag(ALLOW_TRAILING_COMMAS)) goto arr_end; + while (*cur != ',') cur--; goto fail_trailing_comma; } if (char_is_space(*cur)) { while (char_is_space(*++cur)); goto arr_val_begin; } - if (unlikely(ext) && (*cur == 'i' || *cur == 'I' || *cur == 'N')) { - val_incr(); - ctn_len++; - if (read_inf_or_nan(false, &cur, pre, val)) goto arr_val_end; - goto fail_character; - } - if (has_flag(ALLOW_COMMENTS)) { - if (skip_spaces_and_comments(&cur)) goto arr_val_begin; - if (byte_match_2(cur, "/*")) goto fail_comment; - } goto fail_character; arr_val_end: @@ -5213,13 +6270,15 @@ static_inline yyjson_doc *read_root_pretty(u8 *hdr, while (char_is_space(*++cur)); goto arr_val_end; } - if (has_flag(ALLOW_COMMENTS)) { + if (false) { if (skip_spaces_and_comments(&cur)) goto arr_val_end; if (byte_match_2(cur, "/*")) goto fail_comment; } goto fail_character; arr_end: + container_depth--; + /* get parent container */ ctn_parent = (yyjson_val *)(void *)((u8 *)ctn - ctn->uni.ofs); @@ -5239,6 +6298,11 @@ static_inline yyjson_doc *read_root_pretty(u8 *hdr, } obj_begin: + container_depth++; + if (unlikely(container_depth >= YYJSON_READER_CONTAINER_RECURSION_LIMIT)) { + goto fail_recursion; + } + /* push container */ ctn->tag = (((u64)ctn_len + 1) << YYJSON_TAG_BIT) | (ctn->tag & YYJSON_TAG_MASK); @@ -5255,30 +6319,30 @@ static_inline yyjson_doc *read_root_pretty(u8 *hdr, while (true) repeat16({ if (byte_match_2(cur, " ")) cur += 2; else break; - }); + }) #else while (true) repeat16({ if (likely(byte_match_2(cur, " "))) cur += 2; else break; - }); + }) #endif if (likely(*cur == '"')) { val_incr(); ctn_len++; - if (likely(read_string(&cur, end, inv, val, &msg))) goto obj_key_end; + if (likely(read_string(&cur, end, val, &msg))) goto obj_key_end; goto fail_string; } if (likely(*cur == '}')) { cur++; if (likely(ctn_len == 0)) goto obj_end; - if (has_flag(ALLOW_TRAILING_COMMAS)) goto obj_end; + while (*cur != ',') cur--; goto fail_trailing_comma; } if (char_is_space(*cur)) { while (char_is_space(*++cur)); goto obj_key_begin; } - if (has_flag(ALLOW_COMMENTS)) { + if (false) { if (skip_spaces_and_comments(&cur)) goto obj_key_begin; if (byte_match_2(cur, "/*")) goto fail_comment; } @@ -5297,23 +6361,19 @@ static_inline yyjson_doc *read_root_pretty(u8 *hdr, while (char_is_space(*++cur)); goto obj_key_end; } - if (has_flag(ALLOW_COMMENTS)) { - if (skip_spaces_and_comments(&cur)) goto obj_key_end; - if (byte_match_2(cur, "/*")) goto fail_comment; - } goto fail_character; obj_val_begin: if (*cur == '"') { val++; ctn_len++; - if (likely(read_string(&cur, end, inv, val, &msg))) goto obj_val_end; + if (likely(read_string(&cur, end, val, &msg))) goto obj_val_end; goto fail_string; } if (char_is_number(*cur)) { val++; ctn_len++; - if (likely(read_number(&cur, pre, ext, val, &msg))) goto obj_val_end; + if (likely(read_number(&cur, val, &msg))) goto obj_val_end; goto fail_number; } if (*cur == '{') { @@ -5340,24 +6400,11 @@ static_inline yyjson_doc *read_root_pretty(u8 *hdr, val++; ctn_len++; if (likely(read_null(&cur, val))) goto obj_val_end; - if (unlikely(ext)) { - if (read_nan(false, &cur, pre, val)) goto obj_val_end; - } goto fail_literal; } if (char_is_space(*cur)) { while (char_is_space(*++cur)); - goto obj_val_begin; - } - if (unlikely(ext) && (*cur == 'i' || *cur == 'I' || *cur == 'N')) { - val++; - ctn_len++; - if (read_inf_or_nan(false, &cur, pre, val)) goto obj_val_end; - goto fail_character; - } - if (has_flag(ALLOW_COMMENTS)) { - if (skip_spaces_and_comments(&cur)) goto obj_val_begin; - if (byte_match_2(cur, "/*")) goto fail_comment; + goto obj_val_begin; } goto fail_character; @@ -5378,13 +6425,11 @@ static_inline yyjson_doc *read_root_pretty(u8 *hdr, while (char_is_space(*++cur)); goto obj_val_end; } - if (has_flag(ALLOW_COMMENTS)) { - if (skip_spaces_and_comments(&cur)) goto obj_val_end; - if (byte_match_2(cur, "/*")) goto fail_comment; - } goto fail_character; obj_end: + container_depth--; + /* pop container */ ctn_parent = (yyjson_val *)(void *)((u8 *)ctn - ctn->uni.ofs); /* point to the next value */ @@ -5402,19 +6447,22 @@ static_inline yyjson_doc *read_root_pretty(u8 *hdr, doc_end: /* check invalid contents after json document */ - if (unlikely(cur < end) && !has_flag(STOP_WHEN_DONE)) { - if (has_flag(ALLOW_COMMENTS)) skip_spaces_and_comments(&cur); - else while (char_is_space(*cur)) cur++; + if (unlikely(cur < end) && !has_read_flag(STOP_WHEN_DONE)) { + if (false) { + skip_spaces_and_comments(&cur); + if (byte_match_2(cur, "/*")) goto fail_comment; + } else { + while (char_is_space(*cur)) cur++; + } if (unlikely(cur < end)) goto fail_garbage; } - if (pre && *pre) **pre = '\0'; doc = (yyjson_doc *)val_hdr; doc->root = val_hdr + hdr_len; doc->alc = alc; doc->dat_read = (usize)(cur - hdr); doc->val_read = (usize)((val - val_hdr)) - hdr_len + 1; - doc->str_pool = has_flag(INSITU) ? NULL : (char *)hdr; + doc->str_pool = has_read_flag(INSITU) ? NULL : (char *)hdr; return doc; fail_string: @@ -5433,8 +6481,9 @@ static_inline yyjson_doc *read_root_pretty(u8 *hdr, return_err(cur, UNEXPECTED_CHARACTER, "unexpected character"); fail_garbage: return_err(cur, UNEXPECTED_CONTENT, "unexpected content after document"); +fail_recursion: + return_err(cur, RECURSION_DEPTH, "array and object recursion depth exceeded"); -#undef has_flag #undef val_incr #undef return_err } @@ -5447,76 +6496,36 @@ static_inline yyjson_doc *read_root_pretty(u8 *hdr, yyjson_doc *yyjson_read_opts(char *dat, usize len, - yyjson_read_flag flg, const yyjson_alc *alc_ptr, yyjson_read_err *err) { -#define has_flag(_flag) unlikely((flg & YYJSON_READ_##_flag) != 0) - #define return_err(_pos, _code, _msg) do { \ err->pos = (usize)(_pos); \ err->msg = _msg; \ err->code = YYJSON_READ_ERROR_##_code; \ - if (!has_flag(INSITU) && hdr) alc.free(alc.ctx, (void *)hdr); \ + if (!has_read_flag(INSITU) && hdr) alc.free(alc.ctx, (void *)hdr); \ return NULL; \ } while (false) - - yyjson_read_err dummy_err; yyjson_alc alc; yyjson_doc *doc; u8 *hdr = NULL, *end, *cur; - -#if YYJSON_DISABLE_NON_STANDARD - flg &= ~YYJSON_READ_ALLOW_TRAILING_COMMAS; - flg &= ~YYJSON_READ_ALLOW_COMMENTS; - flg &= ~YYJSON_READ_ALLOW_INF_AND_NAN; - flg &= ~YYJSON_READ_ALLOW_INVALID_UNICODE; -#endif - - /* validate input parameters */ - if (!err) err = &dummy_err; - if (likely(!alc_ptr)) { + + if (!alc_ptr) { alc = YYJSON_DEFAULT_ALC; } else { alc = *alc_ptr; } - if (unlikely(!dat)) { - return_err(0, INVALID_PARAMETER, "input data is NULL"); - } - if (unlikely(!len)) { - return_err(0, INVALID_PARAMETER, "input length is 0"); - } - - /* add 4-byte zero padding for input data if necessary */ - if (has_flag(INSITU)) { - hdr = (u8 *)dat; - end = (u8 *)dat + len; - cur = (u8 *)dat; - } else { - if (unlikely(len >= USIZE_MAX - YYJSON_PADDING_SIZE)) { - return_err(0, MEMORY_ALLOCATION, "memory allocation failed"); - } - hdr = (u8 *)alc.malloc(alc.ctx, len + YYJSON_PADDING_SIZE); - if (unlikely(!hdr)) { - return_err(0, MEMORY_ALLOCATION, "memory allocation failed"); - } - end = hdr + len; - cur = hdr; - memcpy(hdr, dat, len); - memset(end, 0, YYJSON_PADDING_SIZE); - } + + hdr = (u8 *)alc.malloc(alc.ctx, len + YYJSON_PADDING_SIZE); + end = hdr + len; + cur = hdr; + memcpy(hdr, dat, len); + memset(end, 0, YYJSON_PADDING_SIZE); /* skip empty contents before json document */ if (unlikely(char_is_space_or_comment(*cur))) { - if (has_flag(ALLOW_COMMENTS)) { - if (!skip_spaces_and_comments(&cur)) { - return_err(cur - hdr, INVALID_COMMENT, - "unclosed multiline comment"); - } - } else { - if (likely(char_is_space(*cur))) { - while (char_is_space(*++cur)); - } + if (likely(char_is_space(*cur))) { + while (char_is_space(*++cur)); } if (unlikely(cur >= end)) { return_err(0, EMPTY_CONTENT, "input data is empty"); @@ -5526,39 +6535,20 @@ yyjson_doc *yyjson_read_opts(char *dat, /* read json document */ if (likely(char_is_container(*cur))) { if (char_is_space(cur[1]) && char_is_space(cur[2])) { - doc = read_root_pretty(hdr, cur, end, alc, flg, err); + doc = read_root_pretty(hdr, cur, end, alc, err); } else { - doc = read_root_minify(hdr, cur, end, alc, flg, err); + doc = read_root_minify(hdr, cur, end, alc, err); } } else { - doc = read_root_single(hdr, cur, end, alc, flg, err); + doc = read_root_single(hdr, cur, end, alc, err); } /* check result */ - if (likely(doc)) { - memset(err, 0, sizeof(yyjson_read_err)); - } else { - /* RFC 8259: JSON text MUST be encoded using UTF-8 */ - if (err->pos == 0 && err->code != YYJSON_READ_ERROR_MEMORY_ALLOCATION) { - if ((hdr[0] == 0xEF && hdr[1] == 0xBB && hdr[2] == 0xBF)) { - err->msg = "byte order mark (BOM) is not supported"; - } else if (len >= 4 && - ((hdr[0] == 0x00 && hdr[1] == 0x00 && - hdr[2] == 0xFE && hdr[3] == 0xFF) || - (hdr[0] == 0xFF && hdr[1] == 0xFE && - hdr[2] == 0x00 && hdr[3] == 0x00))) { - err->msg = "UTF-32 encoding is not supported"; - } else if (len >= 2 && - ((hdr[0] == 0xFE && hdr[1] == 0xFF) || - (hdr[0] == 0xFF && hdr[1] == 0xFE))) { - err->msg = "UTF-16 encoding is not supported"; - } - } - if (!has_flag(INSITU)) alc.free(alc.ctx, (void *)hdr); + if (unlikely(!doc)) { + alc.free(alc.ctx, (void *)hdr); } return doc; -#undef has_flag #undef return_err } @@ -5566,36 +6556,64 @@ yyjson_doc *yyjson_read_file(const char *path, yyjson_read_flag flg, const yyjson_alc *alc_ptr, yyjson_read_err *err) { +#define return_err(_code, _msg) do { \ + err->pos = 0; \ + err->msg = _msg; \ + err->code = YYJSON_READ_ERROR_##_code; \ + return NULL; \ +} while (false) + + + yyjson_doc *doc; + FILE *file; + + + if (unlikely(!path)) return_err(INVALID_PARAMETER, "input path is NULL"); + + file = fopen_readonly(path); + if (unlikely(!file)) return_err(FILE_OPEN, "file opening failed"); + + doc = yyjson_read_fp(file, flg, alc_ptr, err); + fclose(file); + return doc; +#undef return_err +} + +yyjson_doc *yyjson_read_fp(FILE *file, + yyjson_read_flag flg, + const yyjson_alc *alc_ptr, + yyjson_read_err *err) { #define return_err(_code, _msg) do { \ err->pos = 0; \ err->msg = _msg; \ err->code = YYJSON_READ_ERROR_##_code; \ - if (file) fclose(file); \ if (buf) alc.free(alc.ctx, buf); \ return NULL; \ } while (false) - yyjson_read_err dummy_err; + yyjson_alc alc = alc_ptr ? *alc_ptr : YYJSON_DEFAULT_ALC; yyjson_doc *doc; - FILE *file = NULL; - long file_size = 0; + long file_size = 0, file_pos; void *buf = NULL; usize buf_size = 0; /* validate input parameters */ - if (!err) err = &dummy_err; - if (unlikely(!path)) return_err(INVALID_PARAMETER, "input path is NULL"); - - /* open file */ - file = fopen_readonly(path); - if (file == NULL) return_err(FILE_OPEN, "file opening failed"); + + if (unlikely(!file)) return_err(INVALID_PARAMETER, "input file is NULL"); - /* get file size */ - if (fseek(file, 0, SEEK_END) == 0) file_size = ftell(file); - rewind(file); + /* get current position */ + file_pos = ftell(file); + if (file_pos != -1) { + /* get total file size, may fail */ + if (fseek(file, 0, SEEK_END) == 0) file_size = ftell(file); + /* reset to original position, may fail */ + if (fseek(file, file_pos, SEEK_SET) != 0) file_size = 0; + /* get file size from current postion to end */ + if (file_size > 0) file_size -= file_pos; + } /* read file */ if (file_size > 0) { @@ -5626,7 +6644,7 @@ yyjson_doc *yyjson_read_file(const char *path, buf = alc.malloc(alc.ctx, buf_size); if (!buf) return_err(MEMORY_ALLOCATION, "fail to alloc memory"); } else { - tmp = alc.realloc(alc.ctx, buf, buf_size); + tmp = alc.realloc(alc.ctx, buf, buf_size - chunk_now, buf_size); if (!tmp) return_err(MEMORY_ALLOCATION, "fail to alloc memory"); buf = tmp; } @@ -5639,12 +6657,11 @@ yyjson_doc *yyjson_read_file(const char *path, if (chunk_now > chunk_max) chunk_now = chunk_max; } } - fclose(file); /* read JSON */ memset((u8 *)buf + file_size, 0, YYJSON_PADDING_SIZE); flg |= YYJSON_READ_INSITU; - doc = yyjson_read_opts((char *)buf, (usize)file_size, flg, &alc, err); + doc = yyjson_read_opts((char *)buf, (usize)file_size, &alc, err); if (doc) { doc->str_pool = (char *)buf; return doc; @@ -5656,6 +6673,74 @@ yyjson_doc *yyjson_read_file(const char *path, #undef return_err } +const char *yyjson_read_number(const char *dat, + yyjson_val *val, + yyjson_read_flag flg, + const yyjson_alc *alc, + yyjson_read_err *err) { +#define return_err(_pos, _code, _msg) do { \ + err->pos = _pos > hdr ? (usize)(_pos - hdr) : 0; \ + err->msg = _msg; \ + err->code = YYJSON_READ_ERROR_##_code; \ + return NULL; \ +} while (false) + + u8 *hdr = constcast(u8 *)dat, *cur = hdr; + bool raw; /* read number as raw */ + u8 *raw_end; /* raw end for null-terminator */ + u8 **pre; /* previous raw end pointer */ + const char *msg; + + +#if !YYJSON_HAS_IEEE_754 || YYJSON_DISABLE_FAST_FP_CONV + u8 buf[128]; + usize dat_len; +#endif + + + if (unlikely(!dat)) { + return_err(cur, INVALID_PARAMETER, "input data is NULL"); + } + if (unlikely(!val)) { + return_err(cur, INVALID_PARAMETER, "output value is NULL"); + } + +#if !YYJSON_HAS_IEEE_754 || YYJSON_DISABLE_FAST_FP_CONV + if (!alc) alc = &YYJSON_DEFAULT_ALC; + dat_len = strlen(dat); + if (dat_len < sizeof(buf)) { + memcpy(buf, dat, dat_len + 1); + hdr = buf; + cur = hdr; + } else { + hdr = (u8 *)alc->malloc(alc->ctx, dat_len + 1); + cur = hdr; + if (unlikely(!hdr)) { + return_err(cur, MEMORY_ALLOCATION, "memory allocation failed"); + } + memcpy(hdr, dat, dat_len + 1); + } + hdr[dat_len] = 0; +#endif + +#if !YYJSON_HAS_IEEE_754 || YYJSON_DISABLE_FAST_FP_CONV + if (!read_number(&cur, val, &msg)) { + if (dat_len >= sizeof(buf)) alc->free(alc->ctx, hdr); + return_err(cur, INVALID_NUMBER, msg); + } + if (dat_len >= sizeof(buf)) alc->free(alc->ctx, hdr); + if (yyjson_is_raw(val)) val->uni.str = dat; + return dat + (cur - hdr); +#else + if (!read_number(&cur, val, &msg)) { + return_err(cur, INVALID_NUMBER, msg); + } + return (const char *)cur; +#endif + +#undef return_err +} + #endif /* YYJSON_DISABLE_READER */ @@ -5712,10 +6797,10 @@ static_inline u8 *write_u32_len_8(u32 val, u8 *buf) { cc = (ccdd * 5243) >> 19; /* (ccdd / 100) */ bb = aabb - aa * 100; /* (aabb % 100) */ dd = ccdd - cc * 100; /* (ccdd % 100) */ - ((v16 *)buf)[0] = ((const v16 *)digit_table)[aa]; - ((v16 *)buf)[1] = ((const v16 *)digit_table)[bb]; - ((v16 *)buf)[2] = ((const v16 *)digit_table)[cc]; - ((v16 *)buf)[3] = ((const v16 *)digit_table)[dd]; + byte_copy_2(buf + 0, digit_table + aa * 2); + byte_copy_2(buf + 2, digit_table + bb * 2); + byte_copy_2(buf + 4, digit_table + cc * 2); + byte_copy_2(buf + 6, digit_table + dd * 2); return buf + 8; } @@ -5723,8 +6808,8 @@ static_inline u8 *write_u32_len_4(u32 val, u8 *buf) { u32 aa, bb; /* 4 digits: aabb */ aa = (val * 5243) >> 19; /* (val / 100) */ bb = val - aa * 100; /* (val % 100) */ - ((v16 *)buf)[0] = ((const v16 *)digit_table)[aa]; - ((v16 *)buf)[1] = ((const v16 *)digit_table)[bb]; + byte_copy_2(buf + 0, digit_table + aa * 2); + byte_copy_2(buf + 2, digit_table + bb * 2); return buf + 4; } @@ -5733,7 +6818,7 @@ static_inline u8 *write_u32_len_1_8(u32 val, u8 *buf) { if (val < 100) { /* 1-2 digits: aa */ lz = val < 10; /* leading zero: 0 or 1 */ - ((v16 *)buf)[0] = *(const v16 *)(digit_table + (val * 2 + lz)); + byte_copy_2(buf + 0, digit_table + val * 2 + lz); buf -= lz; return buf + 2; @@ -5741,9 +6826,9 @@ static_inline u8 *write_u32_len_1_8(u32 val, u8 *buf) { aa = (val * 5243) >> 19; /* (val / 100) */ bb = val - aa * 100; /* (val % 100) */ lz = aa < 10; /* leading zero: 0 or 1 */ - ((v16 *)buf)[0] = *(const v16 *)(digit_table + (aa * 2 + lz)); + byte_copy_2(buf + 0, digit_table + aa * 2 + lz); buf -= lz; - ((v16 *)buf)[1] = ((const v16 *)digit_table)[bb]; + byte_copy_2(buf + 2, digit_table + bb * 2); return buf + 4; } else if (val < 1000000) { /* 5-6 digits: aabbcc */ @@ -5752,10 +6837,10 @@ static_inline u8 *write_u32_len_1_8(u32 val, u8 *buf) { bb = (bbcc * 5243) >> 19; /* (bbcc / 100) */ cc = bbcc - bb * 100; /* (bbcc % 100) */ lz = aa < 10; /* leading zero: 0 or 1 */ - ((v16 *)buf)[0] = *(const v16 *)(digit_table + (aa * 2 + lz)); + byte_copy_2(buf + 0, digit_table + aa * 2 + lz); buf -= lz; - ((v16 *)buf)[1] = ((const v16 *)digit_table)[bb]; - ((v16 *)buf)[2] = ((const v16 *)digit_table)[cc]; + byte_copy_2(buf + 2, digit_table + bb * 2); + byte_copy_2(buf + 4, digit_table + cc * 2); return buf + 6; } else { /* 7-8 digits: aabbccdd */ @@ -5766,11 +6851,11 @@ static_inline u8 *write_u32_len_1_8(u32 val, u8 *buf) { bb = aabb - aa * 100; /* (aabb % 100) */ dd = ccdd - cc * 100; /* (ccdd % 100) */ lz = aa < 10; /* leading zero: 0 or 1 */ - ((v16 *)buf)[0] = *(const v16 *)(digit_table + (aa * 2 + lz)); + byte_copy_2(buf + 0, digit_table + aa * 2 + lz); buf -= lz; - ((v16 *)buf)[1] = ((const v16 *)digit_table)[bb]; - ((v16 *)buf)[2] = ((const v16 *)digit_table)[cc]; - ((v16 *)buf)[3] = ((const v16 *)digit_table)[dd]; + byte_copy_2(buf + 2, digit_table + bb * 2); + byte_copy_2(buf + 4, digit_table + cc * 2); + byte_copy_2(buf + 6, digit_table + dd * 2); return buf + 8; } } @@ -5784,10 +6869,10 @@ static_inline u8 *write_u64_len_5_8(u32 val, u8 *buf) { bb = (bbcc * 5243) >> 19; /* (bbcc / 100) */ cc = bbcc - bb * 100; /* (bbcc % 100) */ lz = aa < 10; /* leading zero: 0 or 1 */ - ((v16 *)buf)[0] = *(const v16 *)(digit_table + (aa * 2 + lz)); + byte_copy_2(buf + 0, digit_table + aa * 2 + lz); buf -= lz; - ((v16 *)buf)[1] = ((const v16 *)digit_table)[bb]; - ((v16 *)buf)[2] = ((const v16 *)digit_table)[cc]; + byte_copy_2(buf + 2, digit_table + bb * 2); + byte_copy_2(buf + 4, digit_table + cc * 2); return buf + 6; } else { /* 7-8 digits: aabbccdd */ @@ -5798,11 +6883,11 @@ static_inline u8 *write_u64_len_5_8(u32 val, u8 *buf) { bb = aabb - aa * 100; /* (aabb % 100) */ dd = ccdd - cc * 100; /* (ccdd % 100) */ lz = aa < 10; /* leading zero: 0 or 1 */ - ((v16 *)buf)[0] = *(const v16 *)(digit_table + (aa * 2 + lz)); + byte_copy_2(buf + 0, digit_table + aa * 2 + lz); buf -= lz; - ((v16 *)buf)[1] = ((const v16 *)digit_table)[bb]; - ((v16 *)buf)[2] = ((const v16 *)digit_table)[cc]; - ((v16 *)buf)[3] = ((const v16 *)digit_table)[dd]; + byte_copy_2(buf + 2, digit_table + bb * 2); + byte_copy_2(buf + 4, digit_table + cc * 2); + byte_copy_2(buf + 6, digit_table + dd * 2); return buf + 8; } } @@ -5921,9 +7006,9 @@ static_inline u8 *write_u64_len_15_to_17_trim(u8 *buf, u64 sig) { buf[0] = (u8)(a + '0'); buf += a > 0; lz = bb < 10 && a == 0; - ((v16 *)buf)[0] = *(const v16 *)(digit_table + (bb * 2 + lz)); + byte_copy_2(buf + 0, digit_table + bb * 2 + lz); buf -= lz; - ((v16 *)buf)[1] = ((const v16 *)digit_table)[cc]; + byte_copy_2(buf + 2, digit_table + cc * 2); if (ffgghhii) { u32 dd = (ddee * 5243) >> 19; /* (ddee / 100) */ @@ -5932,15 +7017,15 @@ static_inline u8 *write_u64_len_15_to_17_trim(u8 *buf, u64 sig) { u32 hhii = ffgghhii - ffgg * 10000; /* (val % 10000) */ u32 ff = (ffgg * 5243) >> 19; /* (aabb / 100) */ u32 gg = ffgg - ff * 100; /* (aabb % 100) */ - ((v16 *)buf)[2] = ((const v16 *)digit_table)[dd]; - ((v16 *)buf)[3] = ((const v16 *)digit_table)[ee]; - ((v16 *)buf)[4] = ((const v16 *)digit_table)[ff]; - ((v16 *)buf)[5] = ((const v16 *)digit_table)[gg]; + byte_copy_2(buf + 4, digit_table + dd * 2); + byte_copy_2(buf + 6, digit_table + ee * 2); + byte_copy_2(buf + 8, digit_table + ff * 2); + byte_copy_2(buf + 10, digit_table + gg * 2); if (hhii) { u32 hh = (hhii * 5243) >> 19; /* (ccdd / 100) */ u32 ii = hhii - hh * 100; /* (ccdd % 100) */ - ((v16 *)buf)[6] = ((const v16 *)digit_table)[hh]; - ((v16 *)buf)[7] = ((const v16 *)digit_table)[ii]; + byte_copy_2(buf + 12, digit_table + hh * 2); + byte_copy_2(buf + 14, digit_table + ii * 2); tz1 = dec_trailing_zero_table[hh]; tz2 = dec_trailing_zero_table[ii]; tz = ii ? tz2 : (tz1 + 2); @@ -5957,8 +7042,8 @@ static_inline u8 *write_u64_len_15_to_17_trim(u8 *buf, u64 sig) { if (ddee) { u32 dd = (ddee * 5243) >> 19; /* (ddee / 100) */ u32 ee = ddee - dd * 100; /* (ddee % 100) */ - ((v16 *)buf)[2] = ((const v16 *)digit_table)[dd]; - ((v16 *)buf)[3] = ((const v16 *)digit_table)[ee]; + byte_copy_2(buf + 4, digit_table + dd * 2); + byte_copy_2(buf + 6, digit_table + ee * 2); tz1 = dec_trailing_zero_table[dd]; tz2 = dec_trailing_zero_table[ee]; tz = ee ? tz2 : (tz1 + 2); @@ -5981,13 +7066,13 @@ static_inline u8 *write_f64_exp(i32 exp, u8 *buf) { exp = exp < 0 ? -exp : exp; if (exp < 100) { u32 lz = exp < 10; - *(v16 *)&buf[0] = *(const v16 *)(digit_table + ((u32)exp * 2 + lz)); + byte_copy_2(buf + 0, digit_table + (u32)exp * 2 + lz); return buf + 2 - lz; } else { u32 hi = ((u32)exp * 656) >> 16; /* exp / 100 */ u32 lo = (u32)exp - hi * 100; /* exp % 100 */ buf[0] = (u8)((u8)hi + (u8)'0'); - *(v16 *)&buf[1] = *(const v16 *)(digit_table + (lo * 2)); + byte_copy_2(buf + 1, digit_table + lo * 2); return buf + 3; } } @@ -6093,7 +7178,7 @@ static_inline void f64_bin_to_dec(u64 sig_raw, u32 exp_raw, 2. Keep decimal point to indicate the number is floating point. 3. Remove positive sign of exponent part. */ -static_noinline u8 *write_f64_raw(u8 *buf, u64 raw, yyjson_write_flag flg) { +static_inline u8 *write_f64_raw(u8 *buf, u64 raw, yyjson_write_flag flg) { u64 sig_bin, sig_dec, sig_raw; i32 exp_bin, exp_dec, sig_len, dot_pos, i, max; u32 exp_raw, hi, lo; @@ -6107,10 +7192,11 @@ static_noinline u8 *write_f64_raw(u8 *buf, u64 raw, yyjson_write_flag flg) { /* return inf and nan */ if (unlikely(exp_raw == ((u32)1 << F64_EXP_BITS) - 1)) { - if (flg & YYJSON_WRITE_INF_AND_NAN_AS_NULL) { + if (has_write_flag(INF_AND_NAN_AS_NULL)) { byte_copy_4(buf, "null"); return buf + 4; - } else if (flg & YYJSON_WRITE_ALLOW_INF_AND_NAN) { + } + else if (has_write_flag(ALLOW_INF_AND_NAN)) { if (sig_raw == 0) { buf[0] = '-'; buf += sign; @@ -6121,9 +7207,8 @@ static_noinline u8 *write_f64_raw(u8 *buf, u64 raw, yyjson_write_flag flg) { byte_copy_4(buf, "NaN"); return buf + 3; } - } else { - return NULL; } + return NULL; } /* add sign for all finite double value, including 0.0 and inf */ @@ -6196,7 +7281,7 @@ static_noinline u8 *write_f64_raw(u8 *buf, u64 raw, yyjson_write_flag flg) { /* write with scientific notation */ /* such as 1.234e56 */ u8 *end = write_u64_len_15_to_17_trim(buf + 1, sig_dec); - end -= (end == buf + 2); /* remove '.0', e.g. 2.0e34 -> 2e134 */ + end -= (end == buf + 2); /* remove '.0', e.g. 2.0e34 -> 2e34 */ exp_dec += sig_len - 1; hdr[0] = hdr[1]; hdr[1] = '.'; @@ -6233,7 +7318,7 @@ static_noinline u8 *write_f64_raw(u8 *buf, u64 raw, yyjson_write_flag flg) { hi = ((u32)exp_dec * 656) >> 16; /* exp / 100 */ lo = (u32)exp_dec - hi * 100; /* exp % 100 */ buf[0] = (u8)((u8)hi + (u8)'0'); - *(v16 *)&buf[1] = *(const v16 *)(digit_table + (lo * 2)); + byte_copy_2(buf + 1, digit_table + lo * 2); buf += 3; return buf; } @@ -6242,13 +7327,13 @@ static_noinline u8 *write_f64_raw(u8 *buf, u64 raw, yyjson_write_flag flg) { #else /* FP_WRITER */ /** Write a double number (requires 32 bytes buffer). */ -static_noinline u8 *write_f64_raw(u8 *buf, u64 raw, yyjson_write_flag flg) { +static_inline u8 *write_f64_raw(u8 *buf, u64 raw, yyjson_write_flag flg) { /* For IEEE 754, `DBL_DECIMAL_DIG` is 17 for round-trip. For non-IEEE formats, 17 is used to avoid buffer overflow, round-trip is not guaranteed. */ -#if defined(DBL_DECIMAL_DIG) +#if defined(DBL_DECIMAL_DIG) && DBL_DECIMAL_DIG != 17 int dig = DBL_DECIMAL_DIG > 17 ? 17 : DBL_DECIMAL_DIG; #else int dig = 17; @@ -6274,10 +7359,11 @@ static_noinline u8 *write_f64_raw(u8 *buf, u64 raw, yyjson_write_flag flg) { cur += (*cur == '-'); if (unlikely(!digi_is_digit(*cur))) { /* nan, inf, or bad output */ - if (flg & YYJSON_WRITE_INF_AND_NAN_AS_NULL) { + if (has_write_flag(INF_AND_NAN_AS_NULL)) { byte_copy_4(buf, "null"); return buf + 4; - } else if (flg & YYJSON_WRITE_ALLOW_INF_AND_NAN) { + } + else if (has_write_flag(ALLOW_INF_AND_NAN)) { if (*cur == 'i') { byte_copy_8(cur, "Infinity"); cur += 8; @@ -6291,11 +7377,14 @@ static_noinline u8 *write_f64_raw(u8 *buf, u64 raw, yyjson_write_flag flg) { } else { /* finite number */ int i = 0; + bool fp = false; for (; i < len; i++) { - if (buf[i] == ',') { - buf[i] = '.'; - break; - } + if (buf[i] == ',') buf[i] = '.'; + if (digi_is_fp((u8)buf[i])) fp = true; + } + if (!fp) { + buf[len++] = '.'; + buf[len++] = '0'; } } return buf + len; @@ -6563,14 +7652,14 @@ static const u8 esc_single_char_table[512] = { /** Returns the encode table with options. */ static_inline const char_enc_type *get_enc_table_with_flag( yyjson_read_flag flg) { - if (unlikely(flg & YYJSON_WRITE_ESCAPE_UNICODE)) { - if (unlikely(flg & YYJSON_WRITE_ESCAPE_SLASHES)) { + if (has_write_flag(ESCAPE_UNICODE)) { + if (has_write_flag(ESCAPE_SLASHES)) { return enc_table_esc_slash; } else { return enc_table_esc; } } else { - if (unlikely(flg & YYJSON_WRITE_ESCAPE_SLASHES)) { + if (has_write_flag(ESCAPE_SLASHES)) { return enc_table_cpy_slash; } else { return enc_table_cpy; @@ -6584,6 +7673,35 @@ static_inline u8 *write_raw(u8 *cur, const u8 *raw, usize raw_len) { return cur + raw_len; } +/** + Write string no-escape. + @param cur Buffer cursor. + @param str A UTF-8 string, null-terminator is not required. + @param str_len Length of string in bytes. + @return The buffer cursor after string. + */ +static_inline u8 *write_string_noesc(u8 *cur, const u8 *str, usize str_len) { + *cur++ = '"'; + while (str_len >= 16) { + byte_copy_16(cur, str); + cur += 16; + str += 16; + str_len -= 16; + } + while (str_len >= 4) { + byte_copy_4(cur, str); + cur += 4; + str += 4; + str_len -= 4; + } + while (str_len) { + *cur++ = *str++; + str_len -= 1; + } + *cur++ = '"'; + return cur; +} + /** Write UTF-8 string (requires len * 6 + 2 bytes buffer). @param cur Buffer cursor. @@ -6598,7 +7716,7 @@ static_inline u8 *write_string(u8 *cur, bool esc, bool inv, const u8 *str, usize str_len, const char_enc_type *enc_table) { - /* UTF-8 character mask and pattern, see `read_string` for details. */ + /* UTF-8 character mask and pattern, see `read_string()` for details. */ #if YYJSON_ENDIAN == YYJSON_BIG_ENDIAN const u16 b2_mask = 0xE0C0UL; const u16 b2_patt = 0xC080UL; @@ -6626,6 +7744,7 @@ static_inline u8 *write_string(u8 *cur, bool esc, bool inv, const u32 b4_err0 = 0x00000004UL; const u32 b4_err1 = 0x00003003UL; #else + /* this should be evaluated at compile-time */ v16_uni b2_mask_uni = {{ 0xE0, 0xC0 }}; v16_uni b2_patt_uni = {{ 0xC0, 0x80 }}; v16_uni b2_requ_uni = {{ 0x1E, 0x00 }}; @@ -6670,8 +7789,8 @@ static_inline u8 *write_string(u8 *cur, bool esc, bool inv, ) /* The replacement character U+FFFD, used to indicate invalid character. */ - const v32 rep = { 'F', 'F', 'F', 'D' }; - const v32 pre = { '\\', 'u', '0', '0' }; + const v32 rep = {{ 'F', 'F', 'F', 'D' }}; + const v32 pre = {{ '\\', 'u', '0', '0' }}; const u8 *src = str; const u8 *end = str + str_len; @@ -6695,26 +7814,26 @@ static_inline u8 *write_string(u8 *cur, bool esc, bool inv, cur += i; src += i; goto copy_utf8; while (end - src >= 16) { - repeat16_incr(expr_jump); + repeat16_incr(expr_jump) byte_copy_16(cur, src); cur += 16; src += 16; } while (end - src >= 4) { - repeat4_incr(expr_jump); + repeat4_incr(expr_jump) byte_copy_4(cur, src); cur += 4; src += 4; } while (end > src) { - expr_jump(0); + expr_jump(0) *cur++ = *src++; } *cur++ = '"'; return cur; - repeat16_incr(expr_stop); + repeat16_incr(expr_stop) #undef expr_jump #undef expr_stop @@ -6731,16 +7850,27 @@ static_inline u8 *write_string(u8 *cur, bool esc, bool inv, } case CHAR_ENC_CPY_2: { u16 v; +#if YYJSON_DISABLE_UTF8_VALIDATION + byte_copy_2(cur, src); +#else v = byte_load_2(src); if (unlikely(!is_valid_seq_2(v))) goto err_cpy; - byte_copy_2(cur, src); +#endif cur += 2; src += 2; goto copy_utf8; } case CHAR_ENC_CPY_3: { u32 v, tmp; +#if YYJSON_DISABLE_UTF8_VALIDATION + if (likely(src + 4 <= end)) { + byte_copy_4(cur, src); + } else { + byte_copy_2(cur, src); + cur[2] = src[2]; + } +#else if (likely(src + 4 <= end)) { v = byte_load_4(src); if (unlikely(!is_valid_seq_3(v))) goto err_cpy; @@ -6750,22 +7880,26 @@ static_inline u8 *write_string(u8 *cur, bool esc, bool inv, if (unlikely(!is_valid_seq_3(v))) goto err_cpy; byte_copy_4(cur, &v); } +#endif cur += 3; src += 3; goto copy_utf8; } case CHAR_ENC_CPY_4: { u32 v, tmp; +#if YYJSON_DISABLE_UTF8_VALIDATION + byte_copy_4(cur, src); +#else v = byte_load_4(src); if (unlikely(!is_valid_seq_4(v))) goto err_cpy; - byte_copy_4(cur, src); +#endif cur += 4; src += 4; goto copy_utf8; } case CHAR_ENC_ESC_A: { - byte_move_2(cur, &esc_single_char_table[*src * 2]); + byte_copy_2(cur, &esc_single_char_table[*src * 2]); cur += 2; src += 1; goto copy_utf8; @@ -6779,9 +7913,10 @@ static_inline u8 *write_string(u8 *cur, bool esc, bool inv, } case CHAR_ENC_ESC_2: { u16 u, v; +#if !YYJSON_DISABLE_UTF8_VALIDATION v = byte_load_2(src); if (unlikely(!is_valid_seq_2(v))) goto err_esc; - +#endif u = (u16)(((u16)(src[0] & 0x1F) << 6) | ((u16)(src[1] & 0x3F) << 0)); byte_copy_2(cur + 0, &pre); @@ -6794,9 +7929,10 @@ static_inline u8 *write_string(u8 *cur, bool esc, bool inv, case CHAR_ENC_ESC_3: { u16 u; u32 v, tmp; +#if !YYJSON_DISABLE_UTF8_VALIDATION v = byte_load_3(src); if (unlikely(!is_valid_seq_3(v))) goto err_esc; - +#endif u = (u16)(((u16)(src[0] & 0x0F) << 12) | ((u16)(src[1] & 0x3F) << 6) | ((u16)(src[2] & 0x3F) << 0)); @@ -6809,9 +7945,10 @@ static_inline u8 *write_string(u8 *cur, bool esc, bool inv, } case CHAR_ENC_ESC_4: { u32 hi, lo, u, v, tmp; +#if !YYJSON_DISABLE_UTF8_VALIDATION v = byte_load_4(src); if (unlikely(!is_valid_seq_4(v))) goto err_esc; - +#endif u = ((u32)(src[0] & 0x07) << 18) | ((u32)(src[1] & 0x3F) << 12) | ((u32)(src[2] & 0x3F) << 6) | @@ -6869,15 +8006,15 @@ static_inline u8 *write_string(u8 *cur, bool esc, bool inv, /** Write null (requires 8 bytes buffer). */ static_inline u8 *write_null(u8 *cur) { - v64 v = { 'n', 'u', 'l', 'l', ',', '\n', 0, 0 }; + v64 v = {{ 'n', 'u', 'l', 'l', ',', '\n', 0, 0 }}; byte_copy_8(cur, &v); return cur + 4; } /** Write bool (requires 8 bytes buffer). */ static_inline u8 *write_bool(u8 *cur, bool val) { - v64 v0 = { 'f', 'a', 'l', 's', 'e', ',', '\n', 0 }; - v64 v1 = { 't', 'r', 'u', 'e', ',', '\n', 0, 0 }; + v64 v0 = {{ 'f', 'a', 'l', 's', 'e', ',', '\n', 0 }}; + v64 v1 = {{ 't', 'r', 'u', 'e', ',', '\n', 0, 0 }}; if (val) { byte_copy_8(cur, &v1); } else { @@ -6886,15 +8023,27 @@ static_inline u8 *write_bool(u8 *cur, bool val) { return cur + 5 - val; } -/** Write indent (requires level * 4 bytes buffer). */ -static_inline u8 *write_indent(u8 *cur, usize level) { +/** Write indent (requires level x 4 bytes buffer). + Param spaces should not larger than 4. */ +static_inline u8 *write_indent(u8 *cur, usize level, usize spaces) { while (level-- > 0) { byte_copy_4(cur, " "); - cur += 4; + cur += spaces; } return cur; } +/** Write data to file pointer. */ +static bool write_dat_to_fp(FILE *fp, u8 *dat, usize len, + yyjson_write_err *err) { + if (fwrite(dat, len, 1, fp) != 1) { + err->msg = "file writing failed"; + err->code = YYJSON_WRITE_ERROR_FILE_WRITE; + return false; + } + return true; +} + /** Write data to file. */ static bool write_dat_to_file(const char *path, u8 *dat, usize len, yyjson_write_err *err) { @@ -6966,7 +8115,7 @@ static_inline u8 *yyjson_write_single(yyjson_val *val, } while (false) #define check_str_len(_len) do { \ - if ((USIZE_MAX < U64_MAX) && (_len >= (USIZE_MAX - 16) / 6)) \ + if ((sizeof(usize) < 8) && (_len >= (USIZE_MAX - 16) / 6)) \ goto fail_alloc; \ } while (false) @@ -6974,15 +8123,18 @@ static_inline u8 *yyjson_write_single(yyjson_val *val, usize str_len; const u8 *str_ptr; const char_enc_type *enc_table = get_enc_table_with_flag(flg); - bool esc = (flg & YYJSON_WRITE_ESCAPE_UNICODE) != 0; - bool inv = (flg & YYJSON_WRITE_ALLOW_INVALID_UNICODE) != 0; + bool cpy = (enc_table == enc_table_cpy); + bool esc = has_write_flag(ESCAPE_UNICODE) != 0; + bool inv = has_write_flag(ALLOW_INVALID_UNICODE) != 0; + bool newline = has_write_flag(NEWLINE_AT_END) != 0; + const usize end_len = 2; /* '\n' and '\0' */ switch (unsafe_yyjson_get_type(val)) { case YYJSON_TYPE_RAW: str_len = unsafe_yyjson_get_len(val); str_ptr = (const u8 *)unsafe_yyjson_get_str(val); check_str_len(str_len); - incr_len(str_len + 1); + incr_len(str_len + end_len); cur = write_raw(cur, str_ptr, str_len); break; @@ -6990,13 +8142,17 @@ static_inline u8 *yyjson_write_single(yyjson_val *val, str_len = unsafe_yyjson_get_len(val); str_ptr = (const u8 *)unsafe_yyjson_get_str(val); check_str_len(str_len); - incr_len(str_len * 6 + 4); - cur = write_string(cur, esc, inv, str_ptr, str_len, enc_table); - if (unlikely(!cur)) goto fail_str; + incr_len(str_len * 6 + 2 + end_len); + if (likely(cpy) && unsafe_yyjson_get_subtype(val)) { + cur = write_string_noesc(cur, str_ptr, str_len); + } else { + cur = write_string(cur, esc, inv, str_ptr, str_len, enc_table); + if (unlikely(!cur)) goto fail_str; + } break; case YYJSON_TYPE_NUM: - incr_len(32); + incr_len(32 + end_len); cur = write_number(cur, val, flg); if (unlikely(!cur)) goto fail_num; break; @@ -7012,13 +8168,13 @@ static_inline u8 *yyjson_write_single(yyjson_val *val, break; case YYJSON_TYPE_ARR: - incr_len(4); + incr_len(2 + end_len); byte_copy_2(cur, "[]"); cur += 2; break; case YYJSON_TYPE_OBJ: - incr_len(4); + incr_len(2 + end_len); byte_copy_2(cur, "{}"); cur += 2; break; @@ -7027,6 +8183,7 @@ static_inline u8 *yyjson_write_single(yyjson_val *val, goto fail_type; } + if (newline) *cur++ = '\n'; *cur = '\0'; *dat_len = (usize)(cur - hdr); memset(err, 0, sizeof(yyjson_write_err)); @@ -7067,9 +8224,10 @@ static_inline u8 *yyjson_write_minify(const yyjson_val *root, if (unlikely((u8 *)(cur + ext_len) >= (u8 *)ctx)) { \ alc_inc = yyjson_max(alc_len / 2, ext_len); \ alc_inc = size_align_up(alc_inc, sizeof(yyjson_write_ctx)); \ - if (size_add_is_overflow(alc_len, alc_inc)) goto fail_alloc; \ + if ((sizeof(usize) < 8) && size_add_is_overflow(alc_len, alc_inc)) \ + goto fail_alloc; \ alc_len += alc_inc; \ - tmp = (u8 *)alc.realloc(alc.ctx, hdr, alc_len); \ + tmp = (u8 *)alc.realloc(alc.ctx, hdr, alc_len - alc_inc, alc_len); \ if (unlikely(!tmp)) goto fail_alloc; \ ctx_len = (usize)(end - (u8 *)ctx); \ ctx_tmp = (yyjson_write_ctx *)(void *)(tmp + (alc_len - ctx_len)); \ @@ -7082,7 +8240,7 @@ static_inline u8 *yyjson_write_minify(const yyjson_val *root, } while (false) #define check_str_len(_len) do { \ - if ((USIZE_MAX < U64_MAX) && (_len >= (USIZE_MAX - 16) / 6)) \ + if ((sizeof(usize) < 8) && (_len >= (USIZE_MAX - 16) / 6)) \ goto fail_alloc; \ } while (false) @@ -7095,8 +8253,10 @@ static_inline u8 *yyjson_write_minify(const yyjson_val *root, usize alc_len, alc_inc, ctx_len, ext_len, str_len; const u8 *str_ptr; const char_enc_type *enc_table = get_enc_table_with_flag(flg); - bool esc = (flg & YYJSON_WRITE_ESCAPE_UNICODE) != 0; - bool inv = (flg & YYJSON_WRITE_ALLOW_INVALID_UNICODE) != 0; + bool cpy = (enc_table == enc_table_cpy); + bool esc = has_write_flag(ESCAPE_UNICODE) != 0; + bool inv = has_write_flag(ALLOW_INVALID_UNICODE) != 0; + bool newline = has_write_flag(NEWLINE_AT_END) != 0; alc_len = root->uni.ofs / sizeof(yyjson_val); alc_len = alc_len * YYJSON_WRITER_ESTIMATED_MINIFY_RATIO + 64; @@ -7108,7 +8268,7 @@ static_inline u8 *yyjson_write_minify(const yyjson_val *root, ctx = (yyjson_write_ctx *)(void *)end; doc_begin: - val = (yyjson_val *)root; + val = constcast(yyjson_val *)root; val_type = unsafe_yyjson_get_type(val); ctn_obj = (val_type == YYJSON_TYPE_OBJ); ctn_len = unsafe_yyjson_get_len(val) << (u8)ctn_obj; @@ -7123,8 +8283,12 @@ static_inline u8 *yyjson_write_minify(const yyjson_val *root, str_ptr = (const u8 *)unsafe_yyjson_get_str(val); check_str_len(str_len); incr_len(str_len * 6 + 16); - cur = write_string(cur, esc, inv, str_ptr, str_len, enc_table); - if (unlikely(!cur)) goto fail_str; + if (likely(cpy) && unsafe_yyjson_get_subtype(val)) { + cur = write_string_noesc(cur, str_ptr, str_len); + } else { + cur = write_string(cur, esc, inv, str_ptr, str_len, enc_table); + if (unlikely(!cur)) goto fail_str; + } *cur++ = is_key ? ':' : ','; goto val_end; } @@ -7199,6 +8363,11 @@ static_inline u8 *yyjson_write_minify(const yyjson_val *root, } doc_end: + if (newline) { + incr_len(2); + *(cur - 1) = '\n'; + cur++; + } *--cur = '\0'; *dat_len = (usize)(cur - hdr); memset(err, 0, sizeof(yyjson_write_err)); @@ -7239,9 +8408,10 @@ static_inline u8 *yyjson_write_pretty(const yyjson_val *root, if (unlikely((u8 *)(cur + ext_len) >= (u8 *)ctx)) { \ alc_inc = yyjson_max(alc_len / 2, ext_len); \ alc_inc = size_align_up(alc_inc, sizeof(yyjson_write_ctx)); \ - if (size_add_is_overflow(alc_len, alc_inc)) goto fail_alloc; \ + if ((sizeof(usize) < 8) && size_add_is_overflow(alc_len, alc_inc)) \ + goto fail_alloc; \ alc_len += alc_inc; \ - tmp = (u8 *)alc.realloc(alc.ctx, hdr, alc_len); \ + tmp = (u8 *)alc.realloc(alc.ctx, hdr, alc_len - alc_inc, alc_len); \ if (unlikely(!tmp)) goto fail_alloc; \ ctx_len = (usize)(end - (u8 *)ctx); \ ctx_tmp = (yyjson_write_ctx *)(void *)(tmp + (alc_len - ctx_len)); \ @@ -7254,7 +8424,7 @@ static_inline u8 *yyjson_write_pretty(const yyjson_val *root, } while (false) #define check_str_len(_len) do { \ - if ((USIZE_MAX < U64_MAX) && (_len >= (USIZE_MAX - 16) / 6)) \ + if ((sizeof(usize) < 8) && (_len >= (USIZE_MAX - 16) / 6)) \ goto fail_alloc; \ } while (false) @@ -7267,8 +8437,11 @@ static_inline u8 *yyjson_write_pretty(const yyjson_val *root, usize alc_len, alc_inc, ctx_len, ext_len, str_len, level; const u8 *str_ptr; const char_enc_type *enc_table = get_enc_table_with_flag(flg); - bool esc = (flg & YYJSON_WRITE_ESCAPE_UNICODE) != 0; - bool inv = (flg & YYJSON_WRITE_ALLOW_INVALID_UNICODE) != 0; + bool cpy = (enc_table == enc_table_cpy); + bool esc = has_write_flag(ESCAPE_UNICODE) != 0; + bool inv = has_write_flag(ALLOW_INVALID_UNICODE) != 0; + usize spaces = has_write_flag(PRETTY_TWO_SPACES) ? 2 : 4; + bool newline = has_write_flag(NEWLINE_AT_END) != 0; alc_len = root->uni.ofs / sizeof(yyjson_val); alc_len = alc_len * YYJSON_WRITER_ESTIMATED_PRETTY_RATIO + 64; @@ -7280,7 +8453,7 @@ static_inline u8 *yyjson_write_pretty(const yyjson_val *root, ctx = (yyjson_write_ctx *)(void *)end; doc_begin: - val = (yyjson_val *)root; + val = constcast(yyjson_val *)root; val_type = unsafe_yyjson_get_type(val); ctn_obj = (val_type == YYJSON_TYPE_OBJ); ctn_len = unsafe_yyjson_get_len(val) << (u8)ctn_obj; @@ -7298,9 +8471,13 @@ static_inline u8 *yyjson_write_pretty(const yyjson_val *root, str_ptr = (const u8 *)unsafe_yyjson_get_str(val); check_str_len(str_len); incr_len(str_len * 6 + 16 + (no_indent ? 0 : level * 4)); - cur = write_indent(cur, no_indent ? 0 : level); - cur = write_string(cur, esc, inv, str_ptr, str_len, enc_table); - if (unlikely(!cur)) goto fail_str; + cur = write_indent(cur, no_indent ? 0 : level, spaces); + if (likely(cpy) && unsafe_yyjson_get_subtype(val)) { + cur = write_string_noesc(cur, str_ptr, str_len); + } else { + cur = write_string(cur, esc, inv, str_ptr, str_len, enc_table); + if (unlikely(!cur)) goto fail_str; + } *cur++ = is_key ? ':' : ','; *cur++ = is_key ? ' ' : '\n'; goto val_end; @@ -7308,7 +8485,7 @@ static_inline u8 *yyjson_write_pretty(const yyjson_val *root, if (val_type == YYJSON_TYPE_NUM) { no_indent = (bool)((u8)ctn_obj & (u8)ctn_len); incr_len(32 + (no_indent ? 0 : level * 4)); - cur = write_indent(cur, no_indent ? 0 : level); + cur = write_indent(cur, no_indent ? 0 : level, spaces); cur = write_number(cur, val, flg); if (unlikely(!cur)) goto fail_num; *cur++ = ','; @@ -7323,7 +8500,7 @@ static_inline u8 *yyjson_write_pretty(const yyjson_val *root, if (unlikely(ctn_len_tmp == 0)) { /* write empty container */ incr_len(16 + (no_indent ? 0 : level * 4)); - cur = write_indent(cur, no_indent ? 0 : level); + cur = write_indent(cur, no_indent ? 0 : level, spaces); *cur++ = (u8)('[' | ((u8)ctn_obj_tmp << 5)); *cur++ = (u8)(']' | ((u8)ctn_obj_tmp << 5)); *cur++ = ','; @@ -7335,7 +8512,7 @@ static_inline u8 *yyjson_write_pretty(const yyjson_val *root, yyjson_write_ctx_set(--ctx, ctn_len, ctn_obj); ctn_len = ctn_len_tmp << (u8)ctn_obj_tmp; ctn_obj = ctn_obj_tmp; - cur = write_indent(cur, no_indent ? 0 : level); + cur = write_indent(cur, no_indent ? 0 : level, spaces); level++; *cur++ = (u8)('[' | ((u8)ctn_obj << 5)); *cur++ = '\n'; @@ -7346,7 +8523,7 @@ static_inline u8 *yyjson_write_pretty(const yyjson_val *root, if (val_type == YYJSON_TYPE_BOOL) { no_indent = (bool)((u8)ctn_obj & (u8)ctn_len); incr_len(16 + (no_indent ? 0 : level * 4)); - cur = write_indent(cur, no_indent ? 0 : level); + cur = write_indent(cur, no_indent ? 0 : level, spaces); cur = write_bool(cur, unsafe_yyjson_get_bool(val)); cur += 2; goto val_end; @@ -7354,7 +8531,7 @@ static_inline u8 *yyjson_write_pretty(const yyjson_val *root, if (val_type == YYJSON_TYPE_NULL) { no_indent = (bool)((u8)ctn_obj & (u8)ctn_len); incr_len(16 + (no_indent ? 0 : level * 4)); - cur = write_indent(cur, no_indent ? 0 : level); + cur = write_indent(cur, no_indent ? 0 : level, spaces); cur = write_null(cur); cur += 2; goto val_end; @@ -7381,7 +8558,7 @@ static_inline u8 *yyjson_write_pretty(const yyjson_val *root, cur -= 2; *cur++ = '\n'; incr_len(level * 4); - cur = write_indent(cur, --level); + cur = write_indent(cur, --level, spaces); *cur++ = (u8)(']' | ((u8)ctn_obj << 5)); if (unlikely((u8 *)ctx >= end)) goto doc_end; yyjson_write_ctx_get(ctx++, &ctn_len, &ctn_obj); @@ -7395,6 +8572,10 @@ static_inline u8 *yyjson_write_pretty(const yyjson_val *root, } doc_end: + if (newline) { + incr_len(2); + *cur++ = '\n'; + } *cur = '\0'; *dat_len = (usize)(cur - hdr); memset(err, 0, sizeof(yyjson_write_err)); @@ -7422,16 +8603,11 @@ char *yyjson_val_write_opts(const yyjson_val *val, yyjson_write_err dummy_err; usize dummy_dat_len; yyjson_alc alc = alc_ptr ? *alc_ptr : YYJSON_DEFAULT_ALC; - yyjson_val *root = (yyjson_val *)val; + yyjson_val *root = constcast(yyjson_val *)val; err = err ? err : &dummy_err; dat_len = dat_len ? dat_len : &dummy_dat_len; -#if YYJSON_DISABLE_NON_STANDARD - flg &= ~YYJSON_WRITE_ALLOW_INF_AND_NAN; - flg &= ~YYJSON_WRITE_ALLOW_INVALID_UNICODE; -#endif - if (unlikely(!root)) { *dat_len = 0; err->msg = "input JSON is NULL"; @@ -7441,7 +8617,7 @@ char *yyjson_val_write_opts(const yyjson_val *val, if (!unsafe_yyjson_is_ctn(root) || unsafe_yyjson_get_len(root) == 0) { return (char *)yyjson_write_single(root, flg, alc, dat_len, err); - } else if (flg & YYJSON_WRITE_PRETTY) { + } else if (flg & (YYJSON_WRITE_PRETTY | YYJSON_WRITE_PRETTY_TWO_SPACES)) { return (char *)yyjson_write_pretty(root, flg, alc, dat_len, err); } else { return (char *)yyjson_write_minify(root, flg, alc, dat_len, err); @@ -7465,7 +8641,7 @@ bool yyjson_val_write_file(const char *path, yyjson_write_err dummy_err; u8 *dat; usize dat_len = 0; - yyjson_val *root = (yyjson_val *)val; + yyjson_val *root = constcast(yyjson_val *)val; bool suc; alc_ptr = alc_ptr ? alc_ptr : &YYJSON_DEFAULT_ALC; @@ -7483,6 +8659,32 @@ bool yyjson_val_write_file(const char *path, return suc; } +bool yyjson_val_write_fp(FILE *fp, + const yyjson_val *val, + yyjson_write_flag flg, + const yyjson_alc *alc_ptr, + yyjson_write_err *err) { + yyjson_write_err dummy_err; + u8 *dat; + usize dat_len = 0; + yyjson_val *root = constcast(yyjson_val *)val; + bool suc; + + alc_ptr = alc_ptr ? alc_ptr : &YYJSON_DEFAULT_ALC; + err = err ? err : &dummy_err; + if (unlikely(!fp)) { + err->msg = "input fp is invalid"; + err->code = YYJSON_READ_ERROR_INVALID_PARAMETER; + return false; + } + + dat = (u8 *)yyjson_val_write_opts(root, flg, alc_ptr, &dat_len, err); + if (unlikely(!dat)) return false; + suc = write_dat_to_fp(fp, dat, dat_len, err); + alc_ptr->free(alc_ptr->ctx, dat); + return suc; +} + bool yyjson_write_file(const char *path, const yyjson_doc *doc, yyjson_write_flag flg, @@ -7492,6 +8694,15 @@ bool yyjson_write_file(const char *path, return yyjson_val_write_file(path, root, flg, alc_ptr, err); } +bool yyjson_write_fp(FILE *fp, + const yyjson_doc *doc, + yyjson_write_flag flg, + const yyjson_alc *alc_ptr, + yyjson_write_err *err) { + yyjson_val *root = doc ? doc->root : NULL; + return yyjson_val_write_fp(fp, root, flg, alc_ptr, err); +} + /*============================================================================== @@ -7519,6 +8730,21 @@ static_inline void yyjson_mut_write_ctx_get(yyjson_mut_write_ctx *ctx, *ctn = ctx->ctn; } +/** Get the estimated number of values for the mutable JSON document. */ +static_inline usize yyjson_mut_doc_estimated_val_num( + const yyjson_mut_doc *doc) { + usize sum = 0; + yyjson_val_chunk *chunk = doc->val_pool.chunks; + while (chunk) { + sum += chunk->chunk_size / sizeof(yyjson_mut_val) - 1; + if (chunk == doc->val_pool.chunks) { + sum -= (usize)(doc->val_pool.end - doc->val_pool.cur); + } + chunk = chunk->next; + } + return sum; +} + /** Write single JSON value. */ static_inline u8 *yyjson_mut_write_single(yyjson_mut_val *val, yyjson_write_flag flg, @@ -7531,6 +8757,7 @@ static_inline u8 *yyjson_mut_write_single(yyjson_mut_val *val, /** Write JSON document minify. The root of this document should be a non-empty container. */ static_inline u8 *yyjson_mut_write_minify(const yyjson_mut_val *root, + usize estimated_val_num, yyjson_write_flag flg, yyjson_alc alc, usize *dat_len, @@ -7549,9 +8776,10 @@ static_inline u8 *yyjson_mut_write_minify(const yyjson_mut_val *root, if (unlikely((u8 *)(cur + ext_len) >= (u8 *)ctx)) { \ alc_inc = yyjson_max(alc_len / 2, ext_len); \ alc_inc = size_align_up(alc_inc, sizeof(yyjson_mut_write_ctx)); \ - if (size_add_is_overflow(alc_len, alc_inc)) goto fail_alloc; \ + if ((sizeof(usize) < 8) && size_add_is_overflow(alc_len, alc_inc)) \ + goto fail_alloc; \ alc_len += alc_inc; \ - tmp = (u8 *)alc.realloc(alc.ctx, hdr, alc_len); \ + tmp = (u8 *)alc.realloc(alc.ctx, hdr, alc_len - alc_inc, alc_len); \ if (unlikely(!tmp)) goto fail_alloc; \ ctx_len = (usize)(end - (u8 *)ctx); \ ctx_tmp = (yyjson_mut_write_ctx *)(void *)(tmp + (alc_len - ctx_len)); \ @@ -7564,7 +8792,7 @@ static_inline u8 *yyjson_mut_write_minify(const yyjson_mut_val *root, } while (false) #define check_str_len(_len) do { \ - if ((USIZE_MAX < U64_MAX) && (_len >= (USIZE_MAX - 16) / 6)) \ + if ((sizeof(usize) < 8) && (_len >= (USIZE_MAX - 16) / 6)) \ goto fail_alloc; \ } while (false) @@ -7577,10 +8805,12 @@ static_inline u8 *yyjson_mut_write_minify(const yyjson_mut_val *root, usize alc_len, alc_inc, ctx_len, ext_len, str_len; const u8 *str_ptr; const char_enc_type *enc_table = get_enc_table_with_flag(flg); - bool esc = (flg & YYJSON_WRITE_ESCAPE_UNICODE) != 0; - bool inv = (flg & YYJSON_WRITE_ALLOW_INVALID_UNICODE) != 0; + bool cpy = (enc_table == enc_table_cpy); + bool esc = has_write_flag(ESCAPE_UNICODE) != 0; + bool inv = has_write_flag(ALLOW_INVALID_UNICODE) != 0; + bool newline = has_write_flag(NEWLINE_AT_END) != 0; - alc_len = 0 * YYJSON_WRITER_ESTIMATED_MINIFY_RATIO + 64; + alc_len = estimated_val_num * YYJSON_WRITER_ESTIMATED_MINIFY_RATIO + 64; alc_len = size_align_up(alc_len, sizeof(yyjson_mut_write_ctx)); hdr = (u8 *)alc.malloc(alc.ctx, alc_len); if (!hdr) goto fail_alloc; @@ -7589,7 +8819,7 @@ static_inline u8 *yyjson_mut_write_minify(const yyjson_mut_val *root, ctx = (yyjson_mut_write_ctx *)(void *)end; doc_begin: - val = (yyjson_mut_val *)root; + val = constcast(yyjson_mut_val *)root; val_type = unsafe_yyjson_get_type(val); ctn_obj = (val_type == YYJSON_TYPE_OBJ); ctn_len = unsafe_yyjson_get_len(val) << (u8)ctn_obj; @@ -7606,8 +8836,12 @@ static_inline u8 *yyjson_mut_write_minify(const yyjson_mut_val *root, str_ptr = (const u8 *)unsafe_yyjson_get_str(val); check_str_len(str_len); incr_len(str_len * 6 + 16); - cur = write_string(cur, esc, inv, str_ptr, str_len, enc_table); - if (unlikely(!cur)) goto fail_str; + if (likely(cpy) && unsafe_yyjson_get_subtype(val)) { + cur = write_string_noesc(cur, str_ptr, str_len); + } else { + cur = write_string(cur, esc, inv, str_ptr, str_len, enc_table); + if (unlikely(!cur)) goto fail_str; + } *cur++ = is_key ? ':' : ','; goto val_end; } @@ -7685,6 +8919,11 @@ static_inline u8 *yyjson_mut_write_minify(const yyjson_mut_val *root, } doc_end: + if (newline) { + incr_len(2); + *(cur - 1) = '\n'; + cur++; + } *--cur = '\0'; *dat_len = (usize)(cur - hdr); err->code = YYJSON_WRITE_SUCCESS; @@ -7708,6 +8947,7 @@ static_inline u8 *yyjson_mut_write_minify(const yyjson_mut_val *root, /** Write JSON document pretty. The root of this document should be a non-empty container. */ static_inline u8 *yyjson_mut_write_pretty(const yyjson_mut_val *root, + usize estimated_val_num, yyjson_write_flag flg, yyjson_alc alc, usize *dat_len, @@ -7726,9 +8966,10 @@ static_inline u8 *yyjson_mut_write_pretty(const yyjson_mut_val *root, if (unlikely((u8 *)(cur + ext_len) >= (u8 *)ctx)) { \ alc_inc = yyjson_max(alc_len / 2, ext_len); \ alc_inc = size_align_up(alc_inc, sizeof(yyjson_mut_write_ctx)); \ - if (size_add_is_overflow(alc_len, alc_inc)) goto fail_alloc; \ + if ((sizeof(usize) < 8) && size_add_is_overflow(alc_len, alc_inc)) \ + goto fail_alloc; \ alc_len += alc_inc; \ - tmp = (u8 *)alc.realloc(alc.ctx, hdr, alc_len); \ + tmp = (u8 *)alc.realloc(alc.ctx, hdr, alc_len - alc_inc, alc_len); \ if (unlikely(!tmp)) goto fail_alloc; \ ctx_len = (usize)(end - (u8 *)ctx); \ ctx_tmp = (yyjson_mut_write_ctx *)(void *)(tmp + (alc_len - ctx_len)); \ @@ -7741,7 +8982,7 @@ static_inline u8 *yyjson_mut_write_pretty(const yyjson_mut_val *root, } while (false) #define check_str_len(_len) do { \ - if ((USIZE_MAX < U64_MAX) && (_len >= (USIZE_MAX - 16) / 6)) \ + if ((sizeof(usize) < 8) && (_len >= (USIZE_MAX - 16) / 6)) \ goto fail_alloc; \ } while (false) @@ -7754,10 +8995,13 @@ static_inline u8 *yyjson_mut_write_pretty(const yyjson_mut_val *root, usize alc_len, alc_inc, ctx_len, ext_len, str_len, level; const u8 *str_ptr; const char_enc_type *enc_table = get_enc_table_with_flag(flg); - bool esc = (flg & YYJSON_WRITE_ESCAPE_UNICODE) != 0; - bool inv = (flg & YYJSON_WRITE_ALLOW_INVALID_UNICODE) != 0; + bool cpy = (enc_table == enc_table_cpy); + bool esc = has_write_flag(ESCAPE_UNICODE) != 0; + bool inv = has_write_flag(ALLOW_INVALID_UNICODE) != 0; + usize spaces = has_write_flag(PRETTY_TWO_SPACES) ? 2 : 4; + bool newline = has_write_flag(NEWLINE_AT_END) != 0; - alc_len = 0 * YYJSON_WRITER_ESTIMATED_PRETTY_RATIO + 64; + alc_len = estimated_val_num * YYJSON_WRITER_ESTIMATED_PRETTY_RATIO + 64; alc_len = size_align_up(alc_len, sizeof(yyjson_mut_write_ctx)); hdr = (u8 *)alc.malloc(alc.ctx, alc_len); if (!hdr) goto fail_alloc; @@ -7766,7 +9010,7 @@ static_inline u8 *yyjson_mut_write_pretty(const yyjson_mut_val *root, ctx = (yyjson_mut_write_ctx *)(void *)end; doc_begin: - val = (yyjson_mut_val *)root; + val = constcast(yyjson_mut_val *)root; val_type = unsafe_yyjson_get_type(val); ctn_obj = (val_type == YYJSON_TYPE_OBJ); ctn_len = unsafe_yyjson_get_len(val) << (u8)ctn_obj; @@ -7786,9 +9030,13 @@ static_inline u8 *yyjson_mut_write_pretty(const yyjson_mut_val *root, str_ptr = (const u8 *)unsafe_yyjson_get_str(val); check_str_len(str_len); incr_len(str_len * 6 + 16 + (no_indent ? 0 : level * 4)); - cur = write_indent(cur, no_indent ? 0 : level); - cur = write_string(cur, esc, inv, str_ptr, str_len, enc_table); - if (unlikely(!cur)) goto fail_str; + cur = write_indent(cur, no_indent ? 0 : level, spaces); + if (likely(cpy) && unsafe_yyjson_get_subtype(val)) { + cur = write_string_noesc(cur, str_ptr, str_len); + } else { + cur = write_string(cur, esc, inv, str_ptr, str_len, enc_table); + if (unlikely(!cur)) goto fail_str; + } *cur++ = is_key ? ':' : ','; *cur++ = is_key ? ' ' : '\n'; goto val_end; @@ -7796,7 +9044,7 @@ static_inline u8 *yyjson_mut_write_pretty(const yyjson_mut_val *root, if (val_type == YYJSON_TYPE_NUM) { no_indent = (bool)((u8)ctn_obj & (u8)ctn_len); incr_len(32 + (no_indent ? 0 : level * 4)); - cur = write_indent(cur, no_indent ? 0 : level); + cur = write_indent(cur, no_indent ? 0 : level, spaces); cur = write_number(cur, (yyjson_val *)val, flg); if (unlikely(!cur)) goto fail_num; *cur++ = ','; @@ -7811,7 +9059,7 @@ static_inline u8 *yyjson_mut_write_pretty(const yyjson_mut_val *root, if (unlikely(ctn_len_tmp == 0)) { /* write empty container */ incr_len(16 + (no_indent ? 0 : level * 4)); - cur = write_indent(cur, no_indent ? 0 : level); + cur = write_indent(cur, no_indent ? 0 : level, spaces); *cur++ = (u8)('[' | ((u8)ctn_obj_tmp << 5)); *cur++ = (u8)(']' | ((u8)ctn_obj_tmp << 5)); *cur++ = ','; @@ -7823,7 +9071,7 @@ static_inline u8 *yyjson_mut_write_pretty(const yyjson_mut_val *root, yyjson_mut_write_ctx_set(--ctx, ctn, ctn_len, ctn_obj); ctn_len = ctn_len_tmp << (u8)ctn_obj_tmp; ctn_obj = ctn_obj_tmp; - cur = write_indent(cur, no_indent ? 0 : level); + cur = write_indent(cur, no_indent ? 0 : level, spaces); level++; *cur++ = (u8)('[' | ((u8)ctn_obj << 5)); *cur++ = '\n'; @@ -7836,7 +9084,7 @@ static_inline u8 *yyjson_mut_write_pretty(const yyjson_mut_val *root, if (val_type == YYJSON_TYPE_BOOL) { no_indent = (bool)((u8)ctn_obj & (u8)ctn_len); incr_len(16 + (no_indent ? 0 : level * 4)); - cur = write_indent(cur, no_indent ? 0 : level); + cur = write_indent(cur, no_indent ? 0 : level, spaces); cur = write_bool(cur, unsafe_yyjson_get_bool(val)); cur += 2; goto val_end; @@ -7844,7 +9092,7 @@ static_inline u8 *yyjson_mut_write_pretty(const yyjson_mut_val *root, if (val_type == YYJSON_TYPE_NULL) { no_indent = (bool)((u8)ctn_obj & (u8)ctn_len); incr_len(16 + (no_indent ? 0 : level * 4)); - cur = write_indent(cur, no_indent ? 0 : level); + cur = write_indent(cur, no_indent ? 0 : level, spaces); cur = write_null(cur); cur += 2; goto val_end; @@ -7871,7 +9119,7 @@ static_inline u8 *yyjson_mut_write_pretty(const yyjson_mut_val *root, cur -= 2; *cur++ = '\n'; incr_len(level * 4); - cur = write_indent(cur, --level); + cur = write_indent(cur, --level, spaces); *cur++ = (u8)(']' | ((u8)ctn_obj << 5)); if (unlikely((u8 *)ctx >= end)) goto doc_end; val = ctn->next; @@ -7886,6 +9134,10 @@ static_inline u8 *yyjson_mut_write_pretty(const yyjson_mut_val *root, } doc_end: + if (newline) { + incr_len(2); + *cur++ = '\n'; + } *cur = '\0'; *dat_len = (usize)(cur - hdr); err->code = YYJSON_WRITE_SUCCESS; @@ -7906,24 +9158,20 @@ static_inline u8 *yyjson_mut_write_pretty(const yyjson_mut_val *root, #undef check_str_len } -char *yyjson_mut_val_write_opts(const yyjson_mut_val *val, - yyjson_write_flag flg, - const yyjson_alc *alc_ptr, - usize *dat_len, - yyjson_write_err *err) { +static char *yyjson_mut_write_opts_impl(const yyjson_mut_val *val, + usize estimated_val_num, + yyjson_write_flag flg, + const yyjson_alc *alc_ptr, + usize *dat_len, + yyjson_write_err *err) { yyjson_write_err dummy_err; usize dummy_dat_len; yyjson_alc alc = alc_ptr ? *alc_ptr : YYJSON_DEFAULT_ALC; - yyjson_mut_val *root = (yyjson_mut_val *)val; + yyjson_mut_val *root = constcast(yyjson_mut_val *)val; err = err ? err : &dummy_err; dat_len = dat_len ? dat_len : &dummy_dat_len; -#if YYJSON_DISABLE_NON_STANDARD - flg &= ~YYJSON_WRITE_ALLOW_INF_AND_NAN; - flg &= ~YYJSON_WRITE_ALLOW_INVALID_UNICODE; -#endif - if (unlikely(!root)) { *dat_len = 0; err->msg = "input JSON is NULL"; @@ -7933,20 +9181,39 @@ char *yyjson_mut_val_write_opts(const yyjson_mut_val *val, if (!unsafe_yyjson_is_ctn(root) || unsafe_yyjson_get_len(root) == 0) { return (char *)yyjson_mut_write_single(root, flg, alc, dat_len, err); - } else if (flg & YYJSON_WRITE_PRETTY) { - return (char *)yyjson_mut_write_pretty(root, flg, alc, dat_len, err); + } else if (flg & (YYJSON_WRITE_PRETTY | YYJSON_WRITE_PRETTY_TWO_SPACES)) { + return (char *)yyjson_mut_write_pretty(root, estimated_val_num, + flg, alc, dat_len, err); } else { - return (char *)yyjson_mut_write_minify(root, flg, alc, dat_len, err); + return (char *)yyjson_mut_write_minify(root, estimated_val_num, + flg, alc, dat_len, err); } } +char *yyjson_mut_val_write_opts(const yyjson_mut_val *val, + yyjson_write_flag flg, + const yyjson_alc *alc_ptr, + usize *dat_len, + yyjson_write_err *err) { + return yyjson_mut_write_opts_impl(val, 0, flg, alc_ptr, dat_len, err); +} + char *yyjson_mut_write_opts(const yyjson_mut_doc *doc, yyjson_write_flag flg, const yyjson_alc *alc_ptr, usize *dat_len, yyjson_write_err *err) { - yyjson_mut_val *root = doc ? doc->root : NULL; - return yyjson_mut_val_write_opts(root, flg, alc_ptr, dat_len, err); + yyjson_mut_val *root; + usize estimated_val_num; + if (likely(doc)) { + root = doc->root; + estimated_val_num = yyjson_mut_doc_estimated_val_num(doc); + } else { + root = NULL; + estimated_val_num = 0; + } + return yyjson_mut_write_opts_impl(root, estimated_val_num, + flg, alc_ptr, dat_len, err); } bool yyjson_mut_val_write_file(const char *path, @@ -7957,7 +9224,7 @@ bool yyjson_mut_val_write_file(const char *path, yyjson_write_err dummy_err; u8 *dat; usize dat_len = 0; - yyjson_mut_val *root = (yyjson_mut_val *)val; + yyjson_mut_val *root = constcast(yyjson_mut_val *)val; bool suc; alc_ptr = alc_ptr ? alc_ptr : &YYJSON_DEFAULT_ALC; @@ -7973,7 +9240,32 @@ bool yyjson_mut_val_write_file(const char *path, suc = write_dat_to_file(path, dat, dat_len, err); alc_ptr->free(alc_ptr->ctx, dat); return suc; +} + +bool yyjson_mut_val_write_fp(FILE *fp, + const yyjson_mut_val *val, + yyjson_write_flag flg, + const yyjson_alc *alc_ptr, + yyjson_write_err *err) { + yyjson_write_err dummy_err; + u8 *dat; + usize dat_len = 0; + yyjson_mut_val *root = constcast(yyjson_mut_val *)val; + bool suc; + + alc_ptr = alc_ptr ? alc_ptr : &YYJSON_DEFAULT_ALC; + err = err ? err : &dummy_err; + if (unlikely(!fp)) { + err->msg = "input fp is invalid"; + err->code = YYJSON_WRITE_ERROR_INVALID_PARAMETER; + return false; + } + dat = (u8 *)yyjson_mut_val_write_opts(root, flg, alc_ptr, &dat_len, err); + if (unlikely(!dat)) return false; + suc = write_dat_to_fp(fp, dat, dat_len, err); + alc_ptr->free(alc_ptr->ctx, dat); + return suc; } bool yyjson_mut_write_file(const char *path, @@ -7985,20 +9277,13 @@ bool yyjson_mut_write_file(const char *path, return yyjson_mut_val_write_file(path, root, flg, alc_ptr, err); } -#endif /* YYJSON_DISABLE_WRITER */ - - - -/*============================================================================== - * Compiler Hint End - *============================================================================*/ +bool yyjson_mut_write_fp(FILE *fp, + const yyjson_mut_doc *doc, + yyjson_write_flag flg, + const yyjson_alc *alc_ptr, + yyjson_write_err *err) { + yyjson_mut_val *root = doc ? doc->root : NULL; + return yyjson_mut_val_write_fp(fp, root, flg, alc_ptr, err); +} -#if defined(__clang__) -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# if (__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6) -# pragma GCC diagnostic pop -# endif -#elif defined(_MSC_VER) -# pragma warning(pop) -#endif /* warning suppress end */ +#endif /* YYJSON_DISABLE_WRITER */ diff --git a/include/yyjson/yyjson.h b/include/yyjson/yyjson.h index 9e5b825b..210449d3 100644 --- a/include/yyjson/yyjson.h +++ b/include/yyjson/yyjson.h @@ -1,12 +1,30 @@ /*============================================================================== - * Created by Yaoyuan on 2019/3/9. - * Copyright (C) 2019 Yaoyuan . - * - * Released under the MIT License: - * https://github.com/ibireme/yyjson/blob/master/LICENSE + Copyright (c) 2020 YaoYuan + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. *============================================================================*/ -/** @file yyjson.h */ +/** + @file yyjson.h + @date 2019-03-09 + @author YaoYuan + */ #ifndef YYJSON_H #define YYJSON_H @@ -17,6 +35,7 @@ * Header Files *============================================================================*/ +#include #include #include #include @@ -30,12 +49,14 @@ *============================================================================*/ /* - Define as 1 to disable JSON reader if you don't need to parse JSON. + Define as 1 to disable JSON reader if JSON parsing is not required. This will disable these functions at compile-time: + - yyjson_read() - yyjson_read_opts() - yyjson_read_file() - - yyjson_read() + - yyjson_read_number() + - yyjson_mut_read_number() This will reduce the binary size by about 60%. */ @@ -43,7 +64,7 @@ #endif /* - Define as 1 to disable JSON writer if you don't need to serialize JSON. + Define as 1 to disable JSON writer if JSON serialization is not required. This will disable these functions at compile-time: - yyjson_write() @@ -64,19 +85,35 @@ #ifndef YYJSON_DISABLE_WRITER #endif +/* + Define as 1 to disable JSON Pointer, JSON Patch and JSON Merge Patch supports. + + This will disable these functions at compile-time: + - yyjson_ptr_xxx() + - yyjson_mut_ptr_xxx() + - yyjson_doc_ptr_xxx() + - yyjson_mut_doc_ptr_xxx() + - yyjson_patch() + - yyjson_mut_patch() + - yyjson_merge_patch() + - yyjson_mut_merge_patch() + */ +#ifndef YYJSON_DISABLE_UTILS +#endif + /* Define as 1 to disable the fast floating-point number conversion in yyjson, and use libc's `strtod/snprintf` instead. - This will reduce binary size by about 30%, but significantly slow down - floating-point reading and writing speed. + This will reduce the binary size by about 30%, but significantly slow down the + floating-point read/write speed. */ #ifndef YYJSON_DISABLE_FAST_FP_CONV #endif /* Define as 1 to disable non-standard JSON support at compile-time: - - Reading and writing inf/nan literal, such as 'NaN', '-Infinity'. + - Reading and writing inf/nan literal, such as `NaN`, `-Infinity`. - Single line and multiple line comments. - Single trailing comma at the end of an object or array. - Invalid unicode in string value. @@ -89,18 +126,36 @@ - YYJSON_WRITE_ALLOW_INF_AND_NAN - YYJSON_WRITE_ALLOW_INVALID_UNICODE - This will reduce binary size by about 10%, and increase performance slightly. + This will reduce the binary size by about 10%, and speed up the reading and + writing speed by about 2% to 6%. */ #ifndef YYJSON_DISABLE_NON_STANDARD #endif /* - Define as 1 to disable unaligned memory access if target architecture does not - support unaligned memory access (such as some embedded processors). + Define as 1 to disable UTF-8 validation at compile time. - If this value is not defined, yyjson will perform some automatic detection. - The wrong definition of this option may cause some performance degradation, - but will not cause run-time errors. + If all input strings are guaranteed to be valid UTF-8 encoding (for example, + some language's String object has already validated the encoding), using this + flag can avoid redundant UTF-8 validation in yyjson. + + This flag can speed up the reading and writing speed of non-ASCII encoded + strings by about 3% to 7%. + + Note: If this flag is used while passing in illegal UTF-8 strings, the + following errors may occur: + - Escaped characters may be ignored when parsing JSON strings. + - Ending quotes may be ignored when parsing JSON strings, causing the string + to be concatenated to the next value. + - When accessing `yyjson_mut_val` for serialization, the string ending may be + accessed out of bounds, causing a segmentation fault. + */ +#ifndef YYJSON_DISABLE_UTF8_VALIDATION +#endif + +/* + Define as 1 to indicate that the target architecture does not support unaligned + memory access. Please refer to the comments in the C file for details. */ #ifndef YYJSON_DISABLE_UNALIGNED_MEMORY_ACCESS #endif @@ -137,8 +192,26 @@ /** compiler version (GCC) */ #ifdef __GNUC__ # define YYJSON_GCC_VER __GNUC__ +# if defined(__GNUC_PATCHLEVEL__) +# define yyjson_gcc_available(major, minor, patch) \ + ((__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) \ + >= (major * 10000 + minor * 100 + patch)) +# else +# define yyjson_gcc_available(major, minor, patch) \ + ((__GNUC__ * 10000 + __GNUC_MINOR__ * 100) \ + >= (major * 10000 + minor * 100 + patch)) +# endif #else # define YYJSON_GCC_VER 0 +# define yyjson_gcc_available(major, minor, patch) 0 +#endif + +/** real gcc check */ +#if !defined(__clang__) && !defined(__INTEL_COMPILER) && !defined(__ICC) && \ + defined(__GNUC__) +# define YYJSON_IS_REAL_GCC 1 +#else +# define YYJSON_IS_REAL_GCC 0 #endif /** C version (STDC) */ @@ -173,6 +246,15 @@ # endif #endif +/** compiler feature check (since clang 2.6, icc 17) */ +#ifndef yyjson_has_feature +# ifdef __has_feature +# define yyjson_has_feature(x) __has_feature(x) +# else +# define yyjson_has_feature(x) 0 +# endif +#endif + /** include check (since gcc 5.0, clang 2.7, icc 16, msvc 2017 15.3) */ #ifndef yyjson_has_include # ifdef __has_include @@ -225,7 +307,8 @@ /** likely for compiler */ #ifndef yyjson_likely -# if yyjson_has_builtin(__builtin_expect) || YYJSON_GCC_VER >= 4 +# if yyjson_has_builtin(__builtin_expect) || \ + (YYJSON_GCC_VER >= 4 && YYJSON_GCC_VER != 5) # define yyjson_likely(expr) __builtin_expect(!!(expr), 1) # else # define yyjson_likely(expr) (expr) @@ -234,13 +317,39 @@ /** unlikely for compiler */ #ifndef yyjson_unlikely -# if yyjson_has_builtin(__builtin_expect) || YYJSON_GCC_VER >= 4 +# if yyjson_has_builtin(__builtin_expect) || \ + (YYJSON_GCC_VER >= 4 && YYJSON_GCC_VER != 5) # define yyjson_unlikely(expr) __builtin_expect(!!(expr), 0) # else # define yyjson_unlikely(expr) (expr) # endif #endif +/** compile-time constant check for compiler */ +#ifndef yyjson_constant_p +# if yyjson_has_builtin(__builtin_constant_p) || (YYJSON_GCC_VER >= 3) +# define YYJSON_HAS_CONSTANT_P 1 +# define yyjson_constant_p(value) __builtin_constant_p(value) +# else +# define YYJSON_HAS_CONSTANT_P 0 +# define yyjson_constant_p(value) 0 +# endif +#endif + +/** deprecate warning */ +#ifndef yyjson_deprecated +# if YYJSON_MSC_VER >= 1400 +# define yyjson_deprecated(msg) __declspec(deprecated(msg)) +# elif yyjson_has_feature(attribute_deprecated_with_message) || \ + (YYJSON_GCC_VER > 4 || (YYJSON_GCC_VER == 4 && __GNUC_MINOR__ >= 5)) +# define yyjson_deprecated(msg) __attribute__((deprecated(msg))) +# elif YYJSON_GCC_VER >= 3 +# define yyjson_deprecated(msg) __attribute__((deprecated)) +# else +# define yyjson_deprecated(msg) +# endif +#endif + /** function export */ #ifndef yyjson_api # if defined(_WIN32) @@ -369,6 +478,18 @@ # endif #endif +/** + Microsoft Visual C++ 6.0 doesn't support converting number from u64 to f64: + error C2520: conversion from unsigned __int64 to double not implemented. + */ +#ifndef YYJSON_U64_TO_F64_NO_IMPL +# if (0 < YYJSON_MSC_VER) && (YYJSON_MSC_VER <= 1200) +# define YYJSON_U64_TO_F64_NO_IMPL 1 +# else +# define YYJSON_U64_TO_F64_NO_IMPL 0 +# endif +#endif + /*============================================================================== @@ -406,16 +527,16 @@ extern "C" { #define YYJSON_VERSION_MAJOR 0 /** The minor version of yyjson. */ -#define YYJSON_VERSION_MINOR 5 +#define YYJSON_VERSION_MINOR 9 /** The patch version of yyjson. */ -#define YYJSON_VERSION_PATCH 1 +#define YYJSON_VERSION_PATCH 0 -/** The version of yyjson in hex: (major << 16) | (minor << 8) | (patch). */ -#define YYJSON_VERSION_HEX 0x000501 +/** The version of yyjson in hex: `(major << 16) | (minor << 8) | (patch)`. */ +#define YYJSON_VERSION_HEX 0x000900 /** The version string of yyjson. */ -#define YYJSON_VERSION_STRING "0.5.1" +#define YYJSON_VERSION_STRING "0.9.0" /** The version of yyjson in hex, same as `YYJSON_VERSION_HEX`. */ yyjson_api uint32_t yyjson_version(void); @@ -426,34 +547,57 @@ yyjson_api uint32_t yyjson_version(void); * JSON Types *============================================================================*/ -/** Type of JSON value (3 bit). */ +/** Type of a JSON value (3 bit). */ typedef uint8_t yyjson_type; +/** No type, invalid. */ #define YYJSON_TYPE_NONE ((uint8_t)0) /* _____000 */ +/** Raw string type, no subtype. */ #define YYJSON_TYPE_RAW ((uint8_t)1) /* _____001 */ +/** Null type: `null` literal, no subtype. */ #define YYJSON_TYPE_NULL ((uint8_t)2) /* _____010 */ +/** Boolean type, subtype: TRUE, FALSE. */ #define YYJSON_TYPE_BOOL ((uint8_t)3) /* _____011 */ +/** Number type, subtype: UINT, SINT, REAL. */ #define YYJSON_TYPE_NUM ((uint8_t)4) /* _____100 */ +/** String type, subtype: NONE, NOESC. */ #define YYJSON_TYPE_STR ((uint8_t)5) /* _____101 */ +/** Array type, no subtype. */ #define YYJSON_TYPE_ARR ((uint8_t)6) /* _____110 */ +/** Object type, no subtype. */ #define YYJSON_TYPE_OBJ ((uint8_t)7) /* _____111 */ -/** Subtype of JSON value (2 bit). */ +/** Subtype of a JSON value (2 bit). */ typedef uint8_t yyjson_subtype; +/** No subtype. */ #define YYJSON_SUBTYPE_NONE ((uint8_t)(0 << 3)) /* ___00___ */ +/** False subtype: `false` literal. */ #define YYJSON_SUBTYPE_FALSE ((uint8_t)(0 << 3)) /* ___00___ */ +/** True subtype: `true` literal. */ #define YYJSON_SUBTYPE_TRUE ((uint8_t)(1 << 3)) /* ___01___ */ +/** Unsigned integer subtype: `uint64_t`. */ #define YYJSON_SUBTYPE_UINT ((uint8_t)(0 << 3)) /* ___00___ */ +/** Signed integer subtype: `int64_t`. */ #define YYJSON_SUBTYPE_SINT ((uint8_t)(1 << 3)) /* ___01___ */ +/** Real number subtype: `double`. */ #define YYJSON_SUBTYPE_REAL ((uint8_t)(2 << 3)) /* ___10___ */ +/** String that do not need to be escaped for writing (internal use). */ +#define YYJSON_SUBTYPE_NOESC ((uint8_t)(1 << 3)) /* ___01___ */ -/** Mask and bits of JSON value. */ +/** The mask used to extract the type of a JSON value. */ #define YYJSON_TYPE_MASK ((uint8_t)0x07) /* _____111 */ +/** The number of bits used by the type. */ #define YYJSON_TYPE_BIT ((uint8_t)3) +/** The mask used to extract the subtype of a JSON value. */ #define YYJSON_SUBTYPE_MASK ((uint8_t)0x18) /* ___11___ */ +/** The number of bits used by the subtype. */ #define YYJSON_SUBTYPE_BIT ((uint8_t)2) +/** The mask used to extract the reserved bits of a JSON value. */ #define YYJSON_RESERVED_MASK ((uint8_t)0xE0) /* 111_____ */ +/** The number of reserved bits. */ #define YYJSON_RESERVED_BIT ((uint8_t)3) +/** The mask used to extract the tag of a JSON value. */ #define YYJSON_TAG_MASK ((uint8_t)0xFF) /* 11111111 */ +/** The number of bits used by the tag. */ #define YYJSON_TAG_BIT ((uint8_t)8) /** Padding size for JSON reader. */ @@ -472,11 +616,11 @@ typedef uint8_t yyjson_subtype; memory allocator. */ typedef struct yyjson_alc { - /** Same as libc's malloc(), should not be NULL. */ + /** Same as libc's malloc(size), should not be NULL. */ void *(*malloc)(void *ctx, size_t size); - /** Same as libc's realloc(), should not be NULL. */ - void *(*realloc)(void *ctx, void *ptr, size_t size); - /** Same as libc's free(), should not be NULL. */ + /** Same as libc's realloc(ptr, size), should not be NULL. */ + void *(*realloc)(void *ctx, void *ptr, size_t old_size, size_t size); + /** Same as libc's free(ptr), should not be NULL. */ void (*free)(void *ctx, void *ptr); /** A context for malloc/realloc/free, can be NULL. */ void *ctx; @@ -485,17 +629,21 @@ typedef struct yyjson_alc { /** A pool allocator uses fixed length pre-allocated memory. - This allocator may used to avoid malloc()/memmove() calls. The pre-allocated - memory should be held by the caller. The upper limit of memory required to - read JSON can be calculated using the yyjson_read_max_memory_usage() function, - but the memory required to write JSON cannot be calculated directly. + This allocator may be used to avoid malloc/realloc calls. The pre-allocated + memory should be held by the caller. The maximum amount of memory required to + read a JSON can be calculated using the `yyjson_read_max_memory_usage()` + function, but the amount of memory required to write a JSON cannot be directly + calculated. - This is not a general-purpose allocator, and should only be used to read or - write single JSON document. + This is not a general-purpose allocator. It is designed to handle a single JSON + data at a time. If it is used for overly complex memory tasks, such as parsing + multiple JSON documents using the same allocator but releasing only a few of + them, it may cause memory fragmentation, resulting in performance degradation + and memory waste. @param alc The allocator to be initialized. If this parameter is NULL, the function will fail and return false. - If `buf` or `size` is invalid, this parameter is left unmodified. + If `buf` or `size` is invalid, this will be set to an empty allocator. @param buf The buffer memory for this allocator. If this parameter is NULL, the function will fail and return false. @param size The size of `buf`, in bytes. @@ -514,9 +662,31 @@ typedef struct yyjson_alc { yyjson_doc *doc = yyjson_read_opts(json, strlen(json), 0, &alc, NULL); // the memory of `doc` is on the stack @endcode + + @warning This Allocator is not thread-safe. */ yyjson_api bool yyjson_alc_pool_init(yyjson_alc *alc, void *buf, size_t size); +/** + A dynamic allocator. + + This allocator has a similar usage to the pool allocator above. However, when + there is not enough memory, this allocator will dynamically request more memory + using libc's `malloc` function, and frees it all at once when it is destroyed. + + @return A new dynamic allocator, or NULL if memory allocation failed. + @note The returned value should be freed with `yyjson_alc_dyn_free()`. + + @warning This Allocator is not thread-safe. + */ +yyjson_api yyjson_alc *yyjson_alc_dyn_new(void); + +/** + Free a dynamic allocator which is created by `yyjson_alc_dyn_new()`. + @param alc The dynamic allocator to be destroyed. + */ +yyjson_api void yyjson_alc_dyn_free(yyjson_alc *alc); + /*============================================================================== @@ -526,7 +696,7 @@ yyjson_api bool yyjson_alc_pool_init(yyjson_alc *alc, void *buf, size_t size); /** An immutable document for reading JSON. This document holds memory for all its JSON values and strings. When it is no - longer used, the user should call yyjson_doc_free() to free its memory. + longer used, the user should call `yyjson_doc_free()` to free its memory. */ typedef struct yyjson_doc yyjson_doc; @@ -540,7 +710,7 @@ typedef struct yyjson_val yyjson_val; /** A mutable document for building JSON. This document holds memory for all its JSON values and strings. When it is no - longer used, the user should call yyjson_mut_doc_free() to free its memory. + longer used, the user should call `yyjson_mut_doc_free()` to free its memory. */ typedef struct yyjson_mut_doc yyjson_mut_doc; @@ -565,17 +735,17 @@ typedef uint32_t yyjson_read_flag; - Read negative integer as int64_t. - Read floating-point number as double with round-to-nearest mode. - Read integer which cannot fit in uint64_t or int64_t as double. - - Report error if real number is infinity. + - Report error if double number is infinity. - Report error if string contains invalid UTF-8 character or BOM. - Report error on trailing commas, comments, inf and nan literals. */ -static const yyjson_read_flag YYJSON_READ_NOFLAG = 0 << 0; +static const yyjson_read_flag YYJSON_READ_NOFLAG = 0; /** Read the input data in-situ. This option allows the reader to modify and use input data to store string values, which can increase reading speed slightly. The caller should hold the input data before free the document. The input data must be padded by at least `YYJSON_PADDING_SIZE` bytes. - For example: "[1,2]" should be "[1,2]\0\0\0\0", length should be 5. */ + For example: `[1,2]` should be `[1,2]\0\0\0\0`, input length should be 5. */ static const yyjson_read_flag YYJSON_READ_INSITU = 1 << 0; /** Stop when done instead of issuing an error if there's additional content @@ -584,7 +754,7 @@ static const yyjson_read_flag YYJSON_READ_INSITU = 1 << 0; static const yyjson_read_flag YYJSON_READ_STOP_WHEN_DONE = 1 << 1; /** Allow single trailing comma at the end of an object or array, - such as [1,2,3,] {"a":1,"b":2,} (non-standard). */ + such as `[1,2,3,]`, `{"a":1,"b":2,}` (non-standard). */ static const yyjson_read_flag YYJSON_READ_ALLOW_TRAILING_COMMAS = 1 << 2; /** Allow C-style single line and multiple line comments (non-standard). */ @@ -594,7 +764,7 @@ static const yyjson_read_flag YYJSON_READ_ALLOW_COMMENTS = 1 << 3; such as 1e999, NaN, inf, -Infinity (non-standard). */ static const yyjson_read_flag YYJSON_READ_ALLOW_INF_AND_NAN = 1 << 4; -/** Read number as raw string (value with YYJSON_TYPE_RAW type), +/** Read all numbers as raw strings (value with `YYJSON_TYPE_RAW` type), inf/nan literal is also read as raw with `ALLOW_INF_AND_NAN` flag. */ static const yyjson_read_flag YYJSON_READ_NUMBER_AS_RAW = 1 << 5; @@ -608,6 +778,12 @@ static const yyjson_read_flag YYJSON_READ_NUMBER_AS_RAW = 1 << 5; risks. */ static const yyjson_read_flag YYJSON_READ_ALLOW_INVALID_UNICODE = 1 << 6; +/** Read big numbers as raw strings. These big numbers include integers that + cannot be represented by `int64_t` and `uint64_t`, and floating-point + numbers that cannot be represented by finite `double`. + The flag will be overridden by `YYJSON_READ_NUMBER_AS_RAW` flag. */ +static const yyjson_read_flag YYJSON_READ_BIGNUM_AS_RAW = 1 << 7; + /** Result code for JSON reader. */ @@ -616,7 +792,7 @@ typedef uint32_t yyjson_read_code; /** Success, no error. */ static const yyjson_read_code YYJSON_READ_SUCCESS = 0; -/** Invalid parameter, such as NULL string or invalid file path. */ +/** Invalid parameter, such as NULL input string or 0 input length. */ static const yyjson_read_code YYJSON_READ_ERROR_INVALID_PARAMETER = 1; /** Memory allocation failure occurs. */ @@ -625,28 +801,28 @@ static const yyjson_read_code YYJSON_READ_ERROR_MEMORY_ALLOCATION = 2; /** Input JSON string is empty. */ static const yyjson_read_code YYJSON_READ_ERROR_EMPTY_CONTENT = 3; -/** Unexpected content after document, such as "[1]#". */ +/** Unexpected content after document, such as `[123]abc`. */ static const yyjson_read_code YYJSON_READ_ERROR_UNEXPECTED_CONTENT = 4; -/** Unexpected ending, such as "[123". */ +/** Unexpected ending, such as `[123`. */ static const yyjson_read_code YYJSON_READ_ERROR_UNEXPECTED_END = 5; -/** Unexpected character inside the document, such as "[#]". */ +/** Unexpected character inside the document, such as `[abc]`. */ static const yyjson_read_code YYJSON_READ_ERROR_UNEXPECTED_CHARACTER = 6; -/** Invalid JSON structure, such as "[1,]". */ +/** Invalid JSON structure, such as `[1,]`. */ static const yyjson_read_code YYJSON_READ_ERROR_JSON_STRUCTURE = 7; /** Invalid comment, such as unclosed multi-line comment. */ static const yyjson_read_code YYJSON_READ_ERROR_INVALID_COMMENT = 8; -/** Invalid number, such as "123.e12", "000". */ +/** Invalid number, such as `123.e12`, `000`. */ static const yyjson_read_code YYJSON_READ_ERROR_INVALID_NUMBER = 9; /** Invalid string, such as invalid escaped character inside a string. */ static const yyjson_read_code YYJSON_READ_ERROR_INVALID_STRING = 10; -/** Invalid JSON literal, such as "truu". */ +/** Invalid JSON literal, such as `truu`. */ static const yyjson_read_code YYJSON_READ_ERROR_LITERAL = 11; /** Failed to open a file. */ @@ -655,6 +831,9 @@ static const yyjson_read_code YYJSON_READ_ERROR_FILE_OPEN = 12; /** Failed to read a file. */ static const yyjson_read_code YYJSON_READ_ERROR_FILE_READ = 13; +/** Document exceeded YYJSON_READER_CONTAINER_RECURSION_LIMIT. */ +static const yyjson_read_code YYJSON_READ_ERROR_RECURSION_DEPTH = 14; + /** Error information for JSON reader. */ typedef struct yyjson_read_err { /** Error code, see `yyjson_read_code` for all possible values. */ @@ -681,18 +860,15 @@ typedef struct yyjson_read_err { the `YYJSON_READ_INSITU` flag. @param len The length of JSON data in bytes. If this parameter is 0, the function will fail and return NULL. - @param flg The JSON read options. - Multiple options can be combined with `|` operator. 0 means no options. @param alc The memory allocator used by JSON reader. Pass NULL to use the libc's default allocator. @param err A pointer to receive error information. Pass NULL if you don't need error information. @return A new JSON document, or NULL if an error occurs. - When it's no longer needed, it should be freed with yyjson_doc_free(). + When it's no longer needed, it should be freed with `yyjson_doc_free()`. */ yyjson_api yyjson_doc *yyjson_read_opts(char *dat, size_t len, - yyjson_read_flag flg, const yyjson_alc *alc, yyjson_read_err *err); @@ -712,7 +888,7 @@ yyjson_api yyjson_doc *yyjson_read_opts(char *dat, @param err A pointer to receive error information. Pass NULL if you don't need error information. @return A new JSON document, or NULL if an error occurs. - When it's no longer needed, it should be freed with yyjson_doc_free(). + When it's no longer needed, it should be freed with `yyjson_doc_free()`. @warning On 32-bit operating system, files larger than 2GB may fail to read. */ @@ -721,6 +897,28 @@ yyjson_api yyjson_doc *yyjson_read_file(const char *path, const yyjson_alc *alc, yyjson_read_err *err); +/** + Read JSON from a file pointer. + + @param fp The file pointer. + The data will be read from the current position of the FILE to the end. + If this fp is NULL or invalid, the function will fail and return NULL. + @param flg The JSON read options. + Multiple options can be combined with `|` operator. 0 means no options. + @param alc The memory allocator used by JSON reader. + Pass NULL to use the libc's default allocator. + @param err A pointer to receive error information. + Pass NULL if you don't need error information. + @return A new JSON document, or NULL if an error occurs. + When it's no longer needed, it should be freed with `yyjson_doc_free()`. + + @warning On 32-bit operating system, files larger than 2GB may fail to read. + */ +yyjson_api yyjson_doc *yyjson_read_fp(FILE *fp, + yyjson_read_flag flg, + const yyjson_alc *alc, + yyjson_read_err *err); + /** Read a JSON string. @@ -733,13 +931,14 @@ yyjson_api yyjson_doc *yyjson_read_file(const char *path, @param flg The JSON read options. Multiple options can be combined with `|` operator. 0 means no options. @return A new JSON document, or NULL if an error occurs. - When it's no longer needed, it should be freed with yyjson_doc_free(). + When it's no longer needed, it should be freed with `yyjson_doc_free()`. */ yyjson_api_inline yyjson_doc *yyjson_read(const char *dat, size_t len, yyjson_read_flag flg) { flg &= ~YYJSON_READ_INSITU; /* const string cannot be modified */ - return yyjson_read_opts((char *)dat, len, flg, NULL, NULL); + return yyjson_read_opts((char *)(void *)(size_t)(const void *)dat, + len, NULL, NULL); } /** @@ -799,6 +998,61 @@ yyjson_api_inline size_t yyjson_read_max_memory_usage(size_t len, return len * mul + pad; } +/** + Read a JSON number. + + This function is thread-safe when data is not modified by other threads. + + @param dat The JSON data (UTF-8 without BOM), null-terminator is required. + If this parameter is NULL, the function will fail and return NULL. + @param val The output value where result is stored. + If this parameter is NULL, the function will fail and return NULL. + The value will hold either UINT or SINT or REAL number; + @param flg The JSON read options. + Multiple options can be combined with `|` operator. 0 means no options. + Supports `YYJSON_READ_NUMBER_AS_RAW` and `YYJSON_READ_ALLOW_INF_AND_NAN`. + @param alc The memory allocator used for long number. + It is only used when the built-in floating point reader is disabled. + Pass NULL to use the libc's default allocator. + @param err A pointer to receive error information. + Pass NULL if you don't need error information. + @return If successful, a pointer to the character after the last character + used in the conversion, NULL if an error occurs. + */ +yyjson_api const char *yyjson_read_number(const char *dat, + yyjson_val *val, + yyjson_read_flag flg, + const yyjson_alc *alc, + yyjson_read_err *err); + +/** + Read a JSON number. + + This function is thread-safe when data is not modified by other threads. + + @param dat The JSON data (UTF-8 without BOM), null-terminator is required. + If this parameter is NULL, the function will fail and return NULL. + @param val The output value where result is stored. + If this parameter is NULL, the function will fail and return NULL. + The value will hold either UINT or SINT or REAL number; + @param flg The JSON read options. + Multiple options can be combined with `|` operator. 0 means no options. + Supports `YYJSON_READ_NUMBER_AS_RAW` and `YYJSON_READ_ALLOW_INF_AND_NAN`. + @param alc The memory allocator used for long number. + It is only used when the built-in floating point reader is disabled. + Pass NULL to use the libc's default allocator. + @param err A pointer to receive error information. + Pass NULL if you don't need error information. + @return If successful, a pointer to the character after the last character + used in the conversion, NULL if an error occurs. + */ +yyjson_api_inline const char *yyjson_mut_read_number(const char *dat, + yyjson_mut_val *val, + yyjson_read_flag flg, + const yyjson_alc *alc, + yyjson_read_err *err) { + return yyjson_read_number(dat, (yyjson_val *)val, flg, alc, err); +} /*============================================================================== @@ -813,7 +1067,7 @@ typedef uint32_t yyjson_write_flag; - Report error on inf or nan number. - Report error on invalid UTF-8 string. - Do not escape unicode or slash. */ -static const yyjson_write_flag YYJSON_WRITE_NOFLAG = 0 << 0; +static const yyjson_write_flag YYJSON_WRITE_NOFLAG = 0; /** Write JSON pretty with 4 space indent. */ static const yyjson_write_flag YYJSON_WRITE_PRETTY = 1 << 0; @@ -836,7 +1090,15 @@ static const yyjson_write_flag YYJSON_WRITE_INF_AND_NAN_AS_NULL = 1 << 4; If `YYJSON_WRITE_ESCAPE_UNICODE` flag is also set, invalid character will be escaped as `U+FFFD` (replacement character). This flag does not affect the performance of correctly encoded strings. */ -static const yyjson_read_flag YYJSON_WRITE_ALLOW_INVALID_UNICODE = 1 << 5; +static const yyjson_write_flag YYJSON_WRITE_ALLOW_INVALID_UNICODE = 1 << 5; + +/** Write JSON pretty with 2 space indent. + This flag will override `YYJSON_WRITE_PRETTY` flag. */ +static const yyjson_write_flag YYJSON_WRITE_PRETTY_TWO_SPACES = 1 << 6; + +/** Adds a newline character `\n` at the end of the JSON. + This can be helpful for text editors or NDJSON. */ +static const yyjson_write_flag YYJSON_WRITE_NEWLINE_AT_END = 1 << 7; @@ -893,8 +1155,8 @@ typedef struct yyjson_write_err { Multiple options can be combined with `|` operator. 0 means no options. @param alc The memory allocator used by JSON writer. Pass NULL to use the libc's default allocator. - @param len A pointer to receive output length in bytes. - Pass NULL if you don't need length information. + @param len A pointer to receive output length in bytes (not including the + null-terminator). Pass NULL if you don't need length information. @param err A pointer to receive error information. Pass NULL if you don't need error information. @return A new JSON string, or NULL if an error occurs. @@ -935,6 +1197,30 @@ yyjson_api bool yyjson_write_file(const char *path, const yyjson_alc *alc, yyjson_write_err *err); +/** + Write a document to file pointer with options. + + @param fp The file pointer. + The data will be written to the current position of the file. + If this fp is NULL or invalid, the function will fail and return false. + @param doc The JSON document. + If this doc is NULL or has no root, the function will fail and return false. + @param flg The JSON write options. + Multiple options can be combined with `|` operator. 0 means no options. + @param alc The memory allocator used by JSON writer. + Pass NULL to use the libc's default allocator. + @param err A pointer to receive error information. + Pass NULL if you don't need error information. + @return true if successful, false if an error occurs. + + @warning On 32-bit operating system, files larger than 2GB may fail to write. + */ +yyjson_api bool yyjson_write_fp(FILE *fp, + const yyjson_doc *doc, + yyjson_write_flag flg, + const yyjson_alc *alc, + yyjson_write_err *err); + /** Write a document to JSON string. @@ -944,8 +1230,8 @@ yyjson_api bool yyjson_write_file(const char *path, If this doc is NULL or has no root, the function will fail and return false. @param flg The JSON write options. Multiple options can be combined with `|` operator. 0 means no options. - @param len A pointer to receive output length in bytes. - Pass NULL if you don't need length information. + @param len A pointer to receive output length in bytes (not including the + null-terminator). Pass NULL if you don't need length information. @return A new JSON string, or NULL if an error occurs. This string is encoded as UTF-8 with a null-terminator. When it's no longer needed, it should be freed with free(). @@ -971,8 +1257,8 @@ yyjson_api_inline char *yyjson_write(const yyjson_doc *doc, Multiple options can be combined with `|` operator. 0 means no options. @param alc The memory allocator used by JSON writer. Pass NULL to use the libc's default allocator. - @param len A pointer to receive output length in bytes. - Pass NULL if you don't need length information. + @param len A pointer to receive output length in bytes (not including the + null-terminator). Pass NULL if you don't need length information. @param err A pointer to receive error information. Pass NULL if you don't need error information. @return A new JSON string, or NULL if an error occurs. @@ -1014,18 +1300,42 @@ yyjson_api bool yyjson_mut_write_file(const char *path, const yyjson_alc *alc, yyjson_write_err *err); +/** + Write a document to file pointer with options. + + @param fp The file pointer. + The data will be written to the current position of the file. + If this fp is NULL or invalid, the function will fail and return false. + @param doc The mutable JSON document. + If this doc is NULL or has no root, the function will fail and return false. + @param flg The JSON write options. + Multiple options can be combined with `|` operator. 0 means no options. + @param alc The memory allocator used by JSON writer. + Pass NULL to use the libc's default allocator. + @param err A pointer to receive error information. + Pass NULL if you don't need error information. + @return true if successful, false if an error occurs. + + @warning On 32-bit operating system, files larger than 2GB may fail to write. + */ +yyjson_api bool yyjson_mut_write_fp(FILE *fp, + const yyjson_mut_doc *doc, + yyjson_write_flag flg, + const yyjson_alc *alc, + yyjson_write_err *err); + /** Write a document to JSON string. This function is thread-safe when: - The `doc` is not is not modified by other threads. + The `doc` is not modified by other threads. @param doc The JSON document. If this doc is NULL or has no root, the function will fail and return false. @param flg The JSON write options. Multiple options can be combined with `|` operator. 0 means no options. - @param len A pointer to receive output length in bytes. - Pass NULL if you don't need length information. + @param len A pointer to receive output length in bytes (not including the + null-terminator). Pass NULL if you don't need length information. @return A new JSON string, or NULL if an error occurs. This string is encoded as UTF-8 with a null-terminator. When it's no longer needed, it should be freed with free(). @@ -1054,8 +1364,8 @@ yyjson_api_inline char *yyjson_mut_write(const yyjson_mut_doc *doc, Multiple options can be combined with `|` operator. 0 means no options. @param alc The memory allocator used by JSON writer. Pass NULL to use the libc's default allocator. - @param len A pointer to receive output length in bytes. - Pass NULL if you don't need length information. + @param len A pointer to receive output length in bytes (not including the + null-terminator). Pass NULL if you don't need length information. @param err A pointer to receive error information. Pass NULL if you don't need error information. @return A new JSON string, or NULL if an error occurs. @@ -1096,6 +1406,30 @@ yyjson_api bool yyjson_val_write_file(const char *path, const yyjson_alc *alc, yyjson_write_err *err); +/** + Write a value to file pointer with options. + + @param fp The file pointer. + The data will be written to the current position of the file. + If this path is NULL or invalid, the function will fail and return false. + @param val The JSON root value. + If this parameter is NULL, the function will fail and return NULL. + @param flg The JSON write options. + Multiple options can be combined with `|` operator. 0 means no options. + @param alc The memory allocator used by JSON writer. + Pass NULL to use the libc's default allocator. + @param err A pointer to receive error information. + Pass NULL if you don't need error information. + @return true if successful, false if an error occurs. + + @warning On 32-bit operating system, files larger than 2GB may fail to write. + */ +yyjson_api bool yyjson_val_write_fp(FILE *fp, + const yyjson_val *val, + yyjson_write_flag flg, + const yyjson_alc *alc, + yyjson_write_err *err); + /** Write a value to JSON string. @@ -1105,8 +1439,8 @@ yyjson_api bool yyjson_val_write_file(const char *path, If this parameter is NULL, the function will fail and return NULL. @param flg The JSON write options. Multiple options can be combined with `|` operator. 0 means no options. - @param len A pointer to receive output length in bytes. - Pass NULL if you don't need length information. + @param len A pointer to receive output length in bytes (not including the + null-terminator). Pass NULL if you don't need length information. @return A new JSON string, or NULL if an error occurs. This string is encoded as UTF-8 with a null-terminator. When it's no longer needed, it should be freed with free(). @@ -1130,8 +1464,8 @@ yyjson_api_inline char *yyjson_val_write(const yyjson_val *val, Multiple options can be combined with `|` operator. 0 means no options. @param alc The memory allocator used by JSON writer. Pass NULL to use the libc's default allocator. - @param len A pointer to receive output length in bytes. - Pass NULL if you don't need length information. + @param len A pointer to receive output length in bytes (not including the + null-terminator). Pass NULL if you don't need length information. @param err A pointer to receive error information. Pass NULL if you don't need error information. @return A new JSON string, or NULL if an error occurs. @@ -1173,18 +1507,42 @@ yyjson_api bool yyjson_mut_val_write_file(const char *path, const yyjson_alc *alc, yyjson_write_err *err); +/** + Write a value to JSON file with options. + + @param fp The file pointer. + The data will be written to the current position of the file. + If this path is NULL or invalid, the function will fail and return false. + @param val The mutable JSON root value. + If this parameter is NULL, the function will fail and return NULL. + @param flg The JSON write options. + Multiple options can be combined with `|` operator. 0 means no options. + @param alc The memory allocator used by JSON writer. + Pass NULL to use the libc's default allocator. + @param err A pointer to receive error information. + Pass NULL if you don't need error information. + @return true if successful, false if an error occurs. + + @warning On 32-bit operating system, files larger than 2GB may fail to write. + */ +yyjson_api bool yyjson_mut_val_write_fp(FILE *fp, + const yyjson_mut_val *val, + yyjson_write_flag flg, + const yyjson_alc *alc, + yyjson_write_err *err); + /** Write a value to JSON string. This function is thread-safe when: - The `val` is not is not modified by other threads. + The `val` is not modified by other threads. @param val The JSON root value. If this parameter is NULL, the function will fail and return NULL. @param flg The JSON write options. Multiple options can be combined with `|` operator. 0 means no options. - @param len A pointer to receive output length in bytes. - Pass NULL if you don't need length information. + @param len A pointer to receive output length in bytes (not including the + null-terminator). Pass NULL if you don't need length information. @return A new JSON string, or NULL if an error occurs. This string is encoded as UTF-8 with a null-terminator. When it's no longer needed, it should be freed with free(). @@ -1207,12 +1565,12 @@ yyjson_api_inline yyjson_val *yyjson_doc_get_root(yyjson_doc *doc); /** Returns read size of input JSON data. Returns 0 if `doc` is NULL. - For example: the read size of "[1,2,3]" is 7 bytes. */ + For example: the read size of `[1,2,3]` is 7 bytes. */ yyjson_api_inline size_t yyjson_doc_get_read_size(yyjson_doc *doc); /** Returns total value count in this JSON document. Returns 0 if `doc` is NULL. - For example: the value count of "[1,2,3]" is 4. */ + For example: the value count of `[1,2,3]` is 4. */ yyjson_api_inline size_t yyjson_doc_get_val_count(yyjson_doc *doc); /** Release the JSON document and free the memory. @@ -1329,6 +1687,10 @@ yyjson_api_inline int yyjson_get_int(yyjson_val *val); Returns 0.0 if `val` is NULL or type is not real(double). */ yyjson_api_inline double yyjson_get_real(yyjson_val *val); +/** Returns the content and typecast to `double` if the value is number. + Returns 0.0 if `val` is NULL or type is not number(uint/sint/real). */ +yyjson_api_inline double yyjson_get_num(yyjson_val *val); + /** Returns the content if the value is string. Returns NULL if `val` is NULL or type is not string. */ yyjson_api_inline const char *yyjson_get_str(yyjson_val *val); @@ -1348,9 +1710,59 @@ yyjson_api_inline bool yyjson_equals_strn(yyjson_val *val, const char *str, size_t len); /** Returns whether two JSON values are equal (deep compare). - Returns false if input is NULL. */ + Returns false if input is NULL. + @note the result may be inaccurate if object has duplicate keys. + @warning This function is recursive and may cause a stack overflow + if the object level is too deep. */ yyjson_api_inline bool yyjson_equals(yyjson_val *lhs, yyjson_val *rhs); +/** Set the value to raw. + Returns false if input is NULL or `val` is object or array. + @warning This will modify the `immutable` value, use with caution. */ +yyjson_api_inline bool yyjson_set_raw(yyjson_val *val, + const char *raw, size_t len); + +/** Set the value to null. + Returns false if input is NULL or `val` is object or array. + @warning This will modify the `immutable` value, use with caution. */ +yyjson_api_inline bool yyjson_set_null(yyjson_val *val); + +/** Set the value to bool. + Returns false if input is NULL or `val` is object or array. + @warning This will modify the `immutable` value, use with caution. */ +yyjson_api_inline bool yyjson_set_bool(yyjson_val *val, bool num); + +/** Set the value to uint. + Returns false if input is NULL or `val` is object or array. + @warning This will modify the `immutable` value, use with caution. */ +yyjson_api_inline bool yyjson_set_uint(yyjson_val *val, uint64_t num); + +/** Set the value to sint. + Returns false if input is NULL or `val` is object or array. + @warning This will modify the `immutable` value, use with caution. */ +yyjson_api_inline bool yyjson_set_sint(yyjson_val *val, int64_t num); + +/** Set the value to int. + Returns false if input is NULL or `val` is object or array. + @warning This will modify the `immutable` value, use with caution. */ +yyjson_api_inline bool yyjson_set_int(yyjson_val *val, int num); + +/** Set the value to real. + Returns false if input is NULL or `val` is object or array. + @warning This will modify the `immutable` value, use with caution. */ +yyjson_api_inline bool yyjson_set_real(yyjson_val *val, double num); + +/** Set the value to string (null-terminated). + Returns false if input is NULL or `val` is object or array. + @warning This will modify the `immutable` value, use with caution. */ +yyjson_api_inline bool yyjson_set_str(yyjson_val *val, const char *str); + +/** Set the value to string (with length). + Returns false if input is NULL or `val` is object or array. + @warning This will modify the `immutable` value, use with caution. */ +yyjson_api_inline bool yyjson_set_strn(yyjson_val *val, + const char *str, size_t len); + /*============================================================================== @@ -1364,7 +1776,7 @@ yyjson_api_inline size_t yyjson_arr_size(yyjson_val *arr); /** Returns the element at the specified position in this array. Returns NULL if array is NULL/empty or the index is out of bounds. @warning This function takes a linear search time if array is not flat. - For example: [1,{},3] is flat, [1,[2],3] is not flat. */ + For example: `[1,{},3]` is flat, `[1,[2],3]` is not flat. */ yyjson_api_inline yyjson_val *yyjson_arr_get(yyjson_val *arr, size_t idx); /** Returns the first element of this array. @@ -1374,7 +1786,7 @@ yyjson_api_inline yyjson_val *yyjson_arr_get_first(yyjson_val *arr); /** Returns the last element of this array. Returns NULL if `arr` is NULL/empty or type is not array. @warning This function takes a linear search time if array is not flat. - For example: [1,{},3] is flat, [1,[2],3] is not flat.*/ + For example: `[1,{},3]` is flat, `[1,[2],3]` is not flat.*/ yyjson_api_inline yyjson_val *yyjson_arr_get_last(yyjson_val *arr); @@ -1389,14 +1801,17 @@ yyjson_api_inline yyjson_val *yyjson_arr_get_last(yyjson_val *arr); @par Example @code yyjson_val *val; - yyjson_arr_iter iter; - yyjson_arr_iter_init(arr, &iter); + yyjson_arr_iter iter = yyjson_arr_iter_with(arr); while ((val = yyjson_arr_iter_next(&iter))) { your_func(val); } @endcode */ -typedef struct yyjson_arr_iter yyjson_arr_iter; +typedef struct yyjson_arr_iter { + size_t idx; /**< next value's index */ + size_t max; /**< maximum index (arr.size) */ + yyjson_val *cur; /**< next value */ +} yyjson_arr_iter; /** Initialize an iterator for this array. @@ -1412,6 +1827,17 @@ typedef struct yyjson_arr_iter yyjson_arr_iter; yyjson_api_inline bool yyjson_arr_iter_init(yyjson_val *arr, yyjson_arr_iter *iter); +/** + Create an iterator with an array , same as `yyjson_arr_iter_init()`. + + @param arr The array to be iterated over. + If this parameter is NULL or not an array, an empty iterator will returned. + @return A new iterator for the array. + + @note The iterator does not need to be destroyed. + */ +yyjson_api_inline yyjson_arr_iter yyjson_arr_iter_with(yyjson_val *arr); + /** Returns whether the iteration has more elements. If `iter` is NULL, this function will return false. @@ -1487,8 +1913,7 @@ yyjson_api_inline yyjson_val *yyjson_obj_getn(yyjson_val *obj, const char *key, @par Example @code yyjson_val *key, *val; - yyjson_obj_iter iter; - yyjson_obj_iter_init(obj, &iter); + yyjson_obj_iter iter = yyjson_obj_iter_with(obj); while ((key = yyjson_obj_iter_next(&iter))) { val = yyjson_obj_iter_get_val(key); your_func(key, val); @@ -1500,14 +1925,18 @@ yyjson_api_inline yyjson_val *yyjson_obj_getn(yyjson_val *obj, const char *key, @code // {"k1":1, "k2": 3, "k3": 3} yyjson_val *key, *val; - yyjson_obj_iter iter; - yyjson_obj_iter_init(obj, &iter); + yyjson_obj_iter iter = yyjson_obj_iter_with(obj); yyjson_val *v1 = yyjson_obj_iter_get(&iter, "k1"); yyjson_val *v3 = yyjson_obj_iter_get(&iter, "k3"); @endcode @see yyjson_obj_iter_get() and yyjson_obj_iter_getn() */ -typedef struct yyjson_obj_iter yyjson_obj_iter; +typedef struct yyjson_obj_iter { + size_t idx; /**< next key's index */ + size_t max; /**< maximum key index (obj.size) */ + yyjson_val *cur; /**< next key */ + yyjson_val *obj; /**< the object being iterated */ +} yyjson_obj_iter; /** Initialize an iterator for this object. @@ -1523,6 +1952,17 @@ typedef struct yyjson_obj_iter yyjson_obj_iter; yyjson_api_inline bool yyjson_obj_iter_init(yyjson_val *obj, yyjson_obj_iter *iter); +/** + Create an iterator with an object, same as `yyjson_obj_iter_init()`. + + @param obj The object to be iterated over. + If this parameter is NULL or not an object, an empty iterator will returned. + @return A new iterator for the object. + + @note The iterator does not need to be destroyed. + */ +yyjson_api_inline yyjson_obj_iter yyjson_obj_iter_with(yyjson_val *obj); + /** Returns whether the iteration has more elements. If `iter` is NULL, this function will return false. @@ -1544,7 +1984,7 @@ yyjson_api_inline yyjson_val *yyjson_obj_iter_get_val(yyjson_val *key); /** Iterates to a specified key and returns the value. - This function does the same thing as yyjson_obj_get(), but is much faster + This function does the same thing as `yyjson_obj_get()`, but is much faster if the ordering of the keys is known at compile-time and you are using the same order to look up the values. If the key exists in this object, then the iterator will stop at the next key, otherwise the iterator will not change and @@ -1563,7 +2003,7 @@ yyjson_api_inline yyjson_val *yyjson_obj_iter_get(yyjson_obj_iter *iter, /** Iterates to a specified key and returns the value. - This function does the same thing as yyjson_obj_getn(), but is much faster + This function does the same thing as `yyjson_obj_getn()`, but is much faster if the ordering of the keys is known at compile-time and you are using the same order to look up the values. If the key exists in this object, then the iterator will stop at the next key, otherwise the iterator will not change and @@ -1619,6 +2059,38 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_get_root(yyjson_mut_doc *doc); yyjson_api_inline void yyjson_mut_doc_set_root(yyjson_mut_doc *doc, yyjson_mut_val *root); +/** + Set the string pool size for a mutable document. + This function does not allocate memory immediately, but uses the size when + the next memory allocation is needed. + + If the caller knows the approximate bytes of strings that the document needs to + store (e.g. copy string with `yyjson_mut_strcpy` function), setting a larger + size can avoid multiple memory allocations and improve performance. + + @param doc The mutable document. + @param len The desired string pool size in bytes (total string length). + @return true if successful, false if size is 0 or overflow. + */ +yyjson_api bool yyjson_mut_doc_set_str_pool_size(yyjson_mut_doc *doc, + size_t len); + +/** + Set the value pool size for a mutable document. + This function does not allocate memory immediately, but uses the size when + the next memory allocation is needed. + + If the caller knows the approximate number of values that the document needs to + store (e.g. create new value with `yyjson_mut_xxx` functions), setting a larger + size can avoid multiple memory allocations and improve performance. + + @param doc The mutable document. + @param count The desired value pool size (number of `yyjson_mut_val`). + @return true if successful, false if size is 0 or overflow. + */ +yyjson_api bool yyjson_mut_doc_set_val_pool_size(yyjson_mut_doc *doc, + size_t count); + /** Release the JSON document and free the memory. After calling this function, the `doc` and all values from the `doc` are no longer available. This function will do nothing if the `doc` is NULL. */ @@ -1630,31 +2102,52 @@ yyjson_api yyjson_mut_doc *yyjson_mut_doc_new(const yyjson_alc *alc); /** Copies and returns a new mutable document from input, returns NULL on error. This makes a `deep-copy` on the immutable document. - If allocator is NULL, the default allocator will be used. */ + If allocator is NULL, the default allocator will be used. + @note `imut_doc` -> `mut_doc`. */ yyjson_api yyjson_mut_doc *yyjson_doc_mut_copy(yyjson_doc *doc, const yyjson_alc *alc); /** Copies and returns a new mutable document from input, returns NULL on error. This makes a `deep-copy` on the mutable document. - If allocator is NULL, the default allocator will be used. */ + If allocator is NULL, the default allocator will be used. + @note `mut_doc` -> `mut_doc`. */ yyjson_api yyjson_mut_doc *yyjson_mut_doc_mut_copy(yyjson_mut_doc *doc, const yyjson_alc *alc); /** Copies and returns a new mutable value from input, returns NULL on error. This makes a `deep-copy` on the immutable value. - The memory was managed by mutable document. */ + The memory was managed by mutable document. + @note `imut_val` -> `mut_val`. */ yyjson_api yyjson_mut_val *yyjson_val_mut_copy(yyjson_mut_doc *doc, yyjson_val *val); -/** Copies and return a new mutable value from input, returns NULL on error, +/** Copies and returns a new mutable value from input, returns NULL on error. This makes a `deep-copy` on the mutable value. The memory was managed by mutable document. - + @note `mut_val` -> `mut_val`. @warning This function is recursive and may cause a stack overflow if the object level is too deep. */ yyjson_api yyjson_mut_val *yyjson_mut_val_mut_copy(yyjson_mut_doc *doc, yyjson_mut_val *val); +/** Copies and returns a new immutable document from input, + returns NULL on error. This makes a `deep-copy` on the mutable document. + The returned document should be freed with `yyjson_doc_free()`. + @note `mut_doc` -> `imut_doc`. + @warning This function is recursive and may cause a stack overflow + if the object level is too deep. */ +yyjson_api yyjson_doc *yyjson_mut_doc_imut_copy(yyjson_mut_doc *doc, + const yyjson_alc *alc); + +/** Copies and returns a new immutable document from input, + returns NULL on error. This makes a `deep-copy` on the mutable value. + The returned document should be freed with `yyjson_doc_free()`. + @note `mut_val` -> `imut_doc`. + @warning This function is recursive and may cause a stack overflow + if the object level is too deep. */ +yyjson_api yyjson_doc *yyjson_mut_val_imut_copy(yyjson_mut_val *val, + const yyjson_alc *alc); + /*============================================================================== @@ -1724,11 +2217,11 @@ yyjson_api_inline bool yyjson_mut_is_ctn(yyjson_mut_val *val); *============================================================================*/ /** Returns the JSON value's type. - Returns YYJSON_TYPE_NONE if `val` is NULL. */ + Returns `YYJSON_TYPE_NONE` if `val` is NULL. */ yyjson_api_inline yyjson_type yyjson_mut_get_type(yyjson_mut_val *val); /** Returns the JSON value's subtype. - Returns YYJSON_SUBTYPE_NONE if `val` is NULL. */ + Returns `YYJSON_SUBTYPE_NONE` if `val` is NULL. */ yyjson_api_inline yyjson_subtype yyjson_mut_get_subtype(yyjson_mut_val *val); /** Returns the JSON value's tag. @@ -1764,6 +2257,10 @@ yyjson_api_inline int yyjson_mut_get_int(yyjson_mut_val *val); Returns 0.0 if `val` is NULL or type is not real(double). */ yyjson_api_inline double yyjson_mut_get_real(yyjson_mut_val *val); +/** Returns the content and typecast to `double` if the value is number. + Returns 0.0 if `val` is NULL or type is not number(uint/sint/real). */ +yyjson_api_inline double yyjson_mut_get_num(yyjson_mut_val *val); + /** Returns the content if the value is string. Returns NULL if `val` is NULL or type is not string. */ yyjson_api_inline const char *yyjson_mut_get_str(yyjson_mut_val *val); @@ -1786,12 +2283,69 @@ yyjson_api_inline bool yyjson_mut_equals_strn(yyjson_mut_val *val, /** Returns whether two JSON values are equal (deep compare). Returns false if input is NULL. - + @note the result may be inaccurate if object has duplicate keys. @warning This function is recursive and may cause a stack overflow if the object level is too deep. */ yyjson_api_inline bool yyjson_mut_equals(yyjson_mut_val *lhs, yyjson_mut_val *rhs); +/** Set the value to raw. + Returns false if input is NULL. + @warning This function should not be used on an existing object or array. */ +yyjson_api_inline bool yyjson_mut_set_raw(yyjson_mut_val *val, + const char *raw, size_t len); + +/** Set the value to null. + Returns false if input is NULL. + @warning This function should not be used on an existing object or array. */ +yyjson_api_inline bool yyjson_mut_set_null(yyjson_mut_val *val); + +/** Set the value to bool. + Returns false if input is NULL. + @warning This function should not be used on an existing object or array. */ +yyjson_api_inline bool yyjson_mut_set_bool(yyjson_mut_val *val, bool num); + +/** Set the value to uint. + Returns false if input is NULL. + @warning This function should not be used on an existing object or array. */ +yyjson_api_inline bool yyjson_mut_set_uint(yyjson_mut_val *val, uint64_t num); + +/** Set the value to sint. + Returns false if input is NULL. + @warning This function should not be used on an existing object or array. */ +yyjson_api_inline bool yyjson_mut_set_sint(yyjson_mut_val *val, int64_t num); + +/** Set the value to int. + Returns false if input is NULL. + @warning This function should not be used on an existing object or array. */ +yyjson_api_inline bool yyjson_mut_set_int(yyjson_mut_val *val, int num); + +/** Set the value to real. + Returns false if input is NULL. + @warning This function should not be used on an existing object or array. */ +yyjson_api_inline bool yyjson_mut_set_real(yyjson_mut_val *val, double num); + +/** Set the value to string (null-terminated). + Returns false if input is NULL. + @warning This function should not be used on an existing object or array. */ +yyjson_api_inline bool yyjson_mut_set_str(yyjson_mut_val *val, const char *str); + +/** Set the value to string (with length). + Returns false if input is NULL. + @warning This function should not be used on an existing object or array. */ +yyjson_api_inline bool yyjson_mut_set_strn(yyjson_mut_val *val, + const char *str, size_t len); + +/** Set the value to array. + Returns false if input is NULL. + @warning This function should not be used on an existing object or array. */ +yyjson_api_inline bool yyjson_mut_set_arr(yyjson_mut_val *val); + +/** Set the value to array. + Returns false if input is NULL. + @warning This function should not be used on an existing object or array. */ +yyjson_api_inline bool yyjson_mut_set_obj(yyjson_mut_val *val); + /*============================================================================== @@ -1919,13 +2473,12 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_arr_get_last(yyjson_mut_val *arr); A mutable JSON array iterator. @warning You should not modify the array while iterating over it, but you can - use yyjson_mut_arr_iter_remove() to remove current value. + use `yyjson_mut_arr_iter_remove()` to remove current value. @par Example @code yyjson_mut_val *val; - yyjson_mut_arr_iter iter; - yyjson_mut_arr_iter_init(arr, &iter); + yyjson_mut_arr_iter iter = yyjson_mut_arr_iter_with(arr); while ((val = yyjson_mut_arr_iter_next(&iter))) { your_func(val); if (your_val_is_unused(val)) { @@ -1934,7 +2487,13 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_arr_get_last(yyjson_mut_val *arr); } @endcode */ -typedef struct yyjson_mut_arr_iter yyjson_mut_arr_iter; +typedef struct yyjson_mut_arr_iter { + size_t idx; /**< next value's index */ + size_t max; /**< maximum index (arr.size) */ + yyjson_mut_val *cur; /**< current value */ + yyjson_mut_val *pre; /**< previous value */ + yyjson_mut_val *arr; /**< the array being iterated */ +} yyjson_mut_arr_iter; /** Initialize an iterator for this array. @@ -1951,10 +2510,22 @@ yyjson_api_inline bool yyjson_mut_arr_iter_init(yyjson_mut_val *arr, yyjson_mut_arr_iter *iter); /** - Returns whether the iteration has more elements. - If `iter` is NULL, this function will return false. + Create an iterator with an array , same as `yyjson_mut_arr_iter_init()`. + + @param arr The array to be iterated over. + If this parameter is NULL or not an array, an empty iterator will returned. + @return A new iterator for the array. + + @note The iterator does not need to be destroyed. */ -yyjson_api_inline bool yyjson_mut_arr_iter_has_next( +yyjson_api_inline yyjson_mut_arr_iter yyjson_mut_arr_iter_with( + yyjson_mut_val *arr); + +/** + Returns whether the iteration has more elements. + If `iter` is NULL, this function will return false. + */ +yyjson_api_inline bool yyjson_mut_arr_iter_has_next( yyjson_mut_arr_iter *iter); /** @@ -2273,7 +2844,7 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_arr_with_double( @warning The input strings are not copied, you should keep these strings unmodified for the lifetime of this JSON document. If these strings will be - modified, you should use yyjson_mut_arr_with_strcpy() instead. + modified, you should use `yyjson_mut_arr_with_strcpy()` instead. @par Example @code @@ -2299,7 +2870,7 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_arr_with_str( @warning The input strings are not copied, you should keep these strings unmodified for the lifetime of this JSON document. If these strings will be - modified, you should use yyjson_mut_arr_with_strncpy() instead. + modified, you should use `yyjson_mut_arr_with_strncpy()` instead. @par Example @code @@ -2460,7 +3031,7 @@ yyjson_api_inline bool yyjson_mut_arr_clear(yyjson_mut_val *arr); /** Rotates values in this array for the given number of times. - For example: [1,2,3,4,5] rotate 2 is [3,4,5,1,2]. + For example: `[1,2,3,4,5]` rotate 2 is `[3,4,5,1,2]`. @param arr The array to be rotated. @param idx Index (or times) to rotate. @warning This function takes a linear search time. @@ -2692,13 +3263,12 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_getn(yyjson_mut_val *obj, A mutable JSON object iterator. @warning You should not modify the object while iterating over it, but you can - use yyjson_mut_obj_iter_remove() to remove current value. + use `yyjson_mut_obj_iter_remove()` to remove current value. @par Example @code yyjson_mut_val *key, *val; - yyjson_mut_obj_iter iter; - yyjson_mut_obj_iter_init(obj, &iter); + yyjson_mut_obj_iter iter = yyjson_mut_obj_iter_with(obj); while ((key = yyjson_mut_obj_iter_next(&iter))) { val = yyjson_mut_obj_iter_get_val(key); your_func(key, val); @@ -2713,14 +3283,19 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_getn(yyjson_mut_val *obj, @code // {"k1":1, "k2": 3, "k3": 3} yyjson_mut_val *key, *val; - yyjson_mut_obj_iter iter; - yyjson_mut_obj_iter_init(obj, &iter); + yyjson_mut_obj_iter iter = yyjson_mut_obj_iter_with(obj); yyjson_mut_val *v1 = yyjson_mut_obj_iter_get(&iter, "k1"); yyjson_mut_val *v3 = yyjson_mut_obj_iter_get(&iter, "k3"); @endcode - @see yyjson_mut_obj_iter_get() and yyjson_mut_obj_iter_getn() + @see `yyjson_mut_obj_iter_get()` and `yyjson_mut_obj_iter_getn()` */ -typedef struct yyjson_mut_obj_iter yyjson_mut_obj_iter; +typedef struct yyjson_mut_obj_iter { + size_t idx; /**< next key's index */ + size_t max; /**< maximum key index (obj.size) */ + yyjson_mut_val *cur; /**< current key */ + yyjson_mut_val *pre; /**< previous key */ + yyjson_mut_val *obj; /**< the object being iterated */ +} yyjson_mut_obj_iter; /** Initialize an iterator for this object. @@ -2736,6 +3311,18 @@ typedef struct yyjson_mut_obj_iter yyjson_mut_obj_iter; yyjson_api_inline bool yyjson_mut_obj_iter_init(yyjson_mut_val *obj, yyjson_mut_obj_iter *iter); +/** + Create an iterator with an object, same as `yyjson_obj_iter_init()`. + + @param obj The object to be iterated over. + If this parameter is NULL or not an object, an empty iterator will returned. + @return A new iterator for the object. + + @note The iterator does not need to be destroyed. + */ +yyjson_api_inline yyjson_mut_obj_iter yyjson_mut_obj_iter_with( + yyjson_mut_val *obj); + /** Returns whether the iteration has more elements. If `iter` is NULL, this function will return false. @@ -2758,7 +3345,7 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_iter_get_val( yyjson_mut_val *key); /** - Removes and returns current key-value pair in the iteration. + Removes current key-value pair in the iteration, returns the removed value. If `iter` is NULL, this function will return NULL. */ yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_iter_remove( @@ -2767,7 +3354,7 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_iter_remove( /** Iterates to a specified key and returns the value. - This function does the same thing as yyjson_mut_obj_get(), but is much faster + This function does the same thing as `yyjson_mut_obj_get()`, but is much faster if the ordering of the keys is known at compile-time and you are using the same order to look up the values. If the key exists in this object, then the iterator will stop at the next key, otherwise the iterator will not change and @@ -2786,7 +3373,7 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_iter_get( /** Iterates to a specified key and returns the value. - This function does the same thing as yyjson_mut_obj_getn(), but is much faster + This function does the same thing as `yyjson_mut_obj_getn()` but is much faster if the ordering of the keys is known at compile-time and you are using the same order to look up the values. If the key exists in this object, then the iterator will stop at the next key, otherwise the iterator will not change and @@ -2885,8 +3472,8 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_with_kv(yyjson_mut_doc *doc, Adds a key-value pair at the end of the object. This function allows duplicated key in one object. @param obj The object to which the new key-value pair is to be added. - @param key The key, should be a string which is created by yyjson_mut_str(), - yyjson_mut_strn(), yyjson_mut_strcpy() or yyjson_mut_strncpy(). + @param key The key, should be a string which is created by `yyjson_mut_str()`, + `yyjson_mut_strn()`, `yyjson_mut_strcpy()` or `yyjson_mut_strncpy()`. @param val The value to add to the object. @return Whether successful. */ @@ -2897,10 +3484,10 @@ yyjson_api_inline bool yyjson_mut_obj_add(yyjson_mut_val *obj, Sets a key-value pair at the end of the object. This function may remove all key-value pairs for the given key before add. @param obj The object to which the new key-value pair is to be added. - @param key The key, should be a string which is created by yyjson_mut_str(), - yyjson_mut_strn(), yyjson_mut_strcpy() or yyjson_mut_strncpy(). + @param key The key, should be a string which is created by `yyjson_mut_str()`, + `yyjson_mut_strn()`, `yyjson_mut_strcpy()` or `yyjson_mut_strncpy()`. @param val The value to add to the object. If this value is null, the behavior - is same as yyjson_mut_obj_remove(). + is same as `yyjson_mut_obj_remove()`. @return Whether successful. */ yyjson_api_inline bool yyjson_mut_obj_put(yyjson_mut_val *obj, @@ -2911,8 +3498,8 @@ yyjson_api_inline bool yyjson_mut_obj_put(yyjson_mut_val *obj, Inserts a key-value pair to the object at the given position. This function allows duplicated key in one object. @param obj The object to which the new key-value pair is to be added. - @param key The key, should be a string which is created by yyjson_mut_str(), - yyjson_mut_strn(), yyjson_mut_strcpy() or yyjson_mut_strncpy(). + @param key The key, should be a string which is created by `yyjson_mut_str()`, + `yyjson_mut_strn()`, `yyjson_mut_strcpy()` or `yyjson_mut_strncpy()`. @param val The value to add to the object. @param idx The index to which to insert the new pair. @return Whether successful. @@ -2975,7 +3562,8 @@ yyjson_api_inline bool yyjson_mut_obj_replace(yyjson_mut_val *obj, /** Rotates key-value pairs in the object for the given number of times. - For example: {"a":1,"b":2,"c":3,"d":4} rotate 1 is {"b":2,"c":3,"d":4,"a":1}. + For example: `{"a":1,"b":2,"c":3,"d":4}` rotate 1 is + `{"b":2,"c":3,"d":4,"a":1}`. @param obj The object to be rotated. @param idx Index (or times) to rotate. @return Whether successful. @@ -2994,7 +3582,7 @@ yyjson_api_inline bool yyjson_mut_obj_rotate(yyjson_mut_val *obj, The `key` should be a null-terminated UTF-8 string. This function allows duplicated key in one object. - @warning The key string are not copied, you should keep the string + @warning The key string is not copied, you should keep the string unmodified for the lifetime of this JSON document. */ yyjson_api_inline bool yyjson_mut_obj_add_null(yyjson_mut_doc *doc, yyjson_mut_val *obj, @@ -3004,7 +3592,7 @@ yyjson_api_inline bool yyjson_mut_obj_add_null(yyjson_mut_doc *doc, The `key` should be a null-terminated UTF-8 string. This function allows duplicated key in one object. - @warning The key string are not copied, you should keep the string + @warning The key string is not copied, you should keep the string unmodified for the lifetime of this JSON document. */ yyjson_api_inline bool yyjson_mut_obj_add_true(yyjson_mut_doc *doc, yyjson_mut_val *obj, @@ -3014,7 +3602,7 @@ yyjson_api_inline bool yyjson_mut_obj_add_true(yyjson_mut_doc *doc, The `key` should be a null-terminated UTF-8 string. This function allows duplicated key in one object. - @warning The key string are not copied, you should keep the string + @warning The key string is not copied, you should keep the string unmodified for the lifetime of this JSON document. */ yyjson_api_inline bool yyjson_mut_obj_add_false(yyjson_mut_doc *doc, yyjson_mut_val *obj, @@ -3024,7 +3612,7 @@ yyjson_api_inline bool yyjson_mut_obj_add_false(yyjson_mut_doc *doc, The `key` should be a null-terminated UTF-8 string. This function allows duplicated key in one object. - @warning The key string are not copied, you should keep the string + @warning The key string is not copied, you should keep the string unmodified for the lifetime of this JSON document. */ yyjson_api_inline bool yyjson_mut_obj_add_bool(yyjson_mut_doc *doc, yyjson_mut_val *obj, @@ -3034,7 +3622,7 @@ yyjson_api_inline bool yyjson_mut_obj_add_bool(yyjson_mut_doc *doc, The `key` should be a null-terminated UTF-8 string. This function allows duplicated key in one object. - @warning The key string are not copied, you should keep the string + @warning The key string is not copied, you should keep the string unmodified for the lifetime of this JSON document. */ yyjson_api_inline bool yyjson_mut_obj_add_uint(yyjson_mut_doc *doc, yyjson_mut_val *obj, @@ -3044,7 +3632,7 @@ yyjson_api_inline bool yyjson_mut_obj_add_uint(yyjson_mut_doc *doc, The `key` should be a null-terminated UTF-8 string. This function allows duplicated key in one object. - @warning The key string are not copied, you should keep the string + @warning The key string is not copied, you should keep the string unmodified for the lifetime of this JSON document. */ yyjson_api_inline bool yyjson_mut_obj_add_sint(yyjson_mut_doc *doc, yyjson_mut_val *obj, @@ -3054,7 +3642,7 @@ yyjson_api_inline bool yyjson_mut_obj_add_sint(yyjson_mut_doc *doc, The `key` should be a null-terminated UTF-8 string. This function allows duplicated key in one object. - @warning The key string are not copied, you should keep the string + @warning The key string is not copied, you should keep the string unmodified for the lifetime of this JSON document. */ yyjson_api_inline bool yyjson_mut_obj_add_int(yyjson_mut_doc *doc, yyjson_mut_val *obj, @@ -3064,7 +3652,7 @@ yyjson_api_inline bool yyjson_mut_obj_add_int(yyjson_mut_doc *doc, The `key` should be a null-terminated UTF-8 string. This function allows duplicated key in one object. - @warning The key string are not copied, you should keep the string + @warning The key string is not copied, you should keep the string unmodified for the lifetime of this JSON document. */ yyjson_api_inline bool yyjson_mut_obj_add_real(yyjson_mut_doc *doc, yyjson_mut_val *obj, @@ -3074,7 +3662,7 @@ yyjson_api_inline bool yyjson_mut_obj_add_real(yyjson_mut_doc *doc, The `key` and `val` should be null-terminated UTF-8 strings. This function allows duplicated key in one object. - @warning The key/value string are not copied, you should keep these strings + @warning The key/value strings are not copied, you should keep these strings unmodified for the lifetime of this JSON document. */ yyjson_api_inline bool yyjson_mut_obj_add_str(yyjson_mut_doc *doc, yyjson_mut_val *obj, @@ -3086,7 +3674,7 @@ yyjson_api_inline bool yyjson_mut_obj_add_str(yyjson_mut_doc *doc, The `len` should be the length of the `val`, in bytes. This function allows duplicated key in one object. - @warning The key/value string are not copied, you should keep these strings + @warning The key/value strings are not copied, you should keep these strings unmodified for the lifetime of this JSON document. */ yyjson_api_inline bool yyjson_mut_obj_add_strn(yyjson_mut_doc *doc, yyjson_mut_val *obj, @@ -3098,7 +3686,7 @@ yyjson_api_inline bool yyjson_mut_obj_add_strn(yyjson_mut_doc *doc, The value string is copied. This function allows duplicated key in one object. - @warning The key string are not copied, you should keep the string + @warning The key string is not copied, you should keep the string unmodified for the lifetime of this JSON document. */ yyjson_api_inline bool yyjson_mut_obj_add_strcpy(yyjson_mut_doc *doc, yyjson_mut_val *obj, @@ -3111,18 +3699,44 @@ yyjson_api_inline bool yyjson_mut_obj_add_strcpy(yyjson_mut_doc *doc, The `len` should be the length of the `val`, in bytes. This function allows duplicated key in one object. - @warning The key/value string are not copied, you should keep these strings + @warning The key strings are not copied, you should keep these strings unmodified for the lifetime of this JSON document. */ yyjson_api_inline bool yyjson_mut_obj_add_strncpy(yyjson_mut_doc *doc, yyjson_mut_val *obj, const char *key, const char *val, size_t len); +/** + Creates and adds a new array to the target object. + The `key` should be a null-terminated UTF-8 string. + This function allows duplicated key in one object. + + @warning The key string is not copied, you should keep these strings + unmodified for the lifetime of this JSON document. + @return The new array, or NULL on error. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_add_arr(yyjson_mut_doc *doc, + yyjson_mut_val *obj, + const char *key); + +/** + Creates and adds a new object to the target object. + The `key` should be a null-terminated UTF-8 string. + This function allows duplicated key in one object. + + @warning The key string is not copied, you should keep these strings + unmodified for the lifetime of this JSON document. + @return The new object, or NULL on error. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_add_obj(yyjson_mut_doc *doc, + yyjson_mut_val *obj, + const char *key); + /** Adds a JSON value at the end of the object. The `key` should be a null-terminated UTF-8 string. This function allows duplicated key in one object. - @warning The key string are not copied, you should keep the string + @warning The key string is not copied, you should keep the string unmodified for the lifetime of this JSON document. */ yyjson_api_inline bool yyjson_mut_obj_add_val(yyjson_mut_doc *doc, yyjson_mut_val *obj, @@ -3148,101 +3762,714 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_remove_str( yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_remove_strn( yyjson_mut_val *obj, const char *key, size_t len); +/** Replaces all matching keys with the new key. + Returns true if at least one key was renamed. + The `key` and `new_key` should be a null-terminated UTF-8 string. + The `new_key` is copied and held by doc. + + @warning This function takes a linear search time. + If `new_key` already exists, it will cause duplicate keys. + */ +yyjson_api_inline bool yyjson_mut_obj_rename_key(yyjson_mut_doc *doc, + yyjson_mut_val *obj, + const char *key, + const char *new_key); + +/** Replaces all matching keys with the new key. + Returns true if at least one key was renamed. + The `key` and `new_key` should be a UTF-8 string, + null-terminator is not required. The `new_key` is copied and held by doc. + + @warning This function takes a linear search time. + If `new_key` already exists, it will cause duplicate keys. + */ +yyjson_api_inline bool yyjson_mut_obj_rename_keyn(yyjson_mut_doc *doc, + yyjson_mut_val *obj, + const char *key, + size_t len, + const char *new_key, + size_t new_len); + /*============================================================================== - * JSON Pointer API + * JSON Pointer API (RFC 6901) * https://tools.ietf.org/html/rfc6901 *============================================================================*/ -/** Get a JSON value with JSON Pointer (RFC 6901). - The `ptr` should be a null-terminated UTF-8 string. - - Returns NULL if there's no matched value. - Returns NULL if `val/ptr` is NULL or `val` is not object. */ -yyjson_api_inline yyjson_val *yyjson_get_pointer(yyjson_val *val, - const char *ptr); +/** JSON Pointer error code. */ +typedef uint32_t yyjson_ptr_code; -/** Get a JSON value with JSON Pointer (RFC 6901). - The `ptr` should be a UTF-8 string, null-terminator is not required. - The `len` should be the length of the `ptr`, in bytes. - - Returns NULL if there's no matched value. - Returns NULL if `val/ptr` is NULL or `val` is not object. */ -yyjson_api_inline yyjson_val *yyjson_get_pointern(yyjson_val *val, - const char *ptr, - size_t len); +/** No JSON pointer error. */ +static const yyjson_ptr_code YYJSON_PTR_ERR_NONE = 0; -/** Get a JSON value with JSON Pointer (RFC 6901). - The `ptr` should be a null-terminated UTF-8 string. - - Returns NULL if there's no matched value. */ -yyjson_api_inline yyjson_val *yyjson_doc_get_pointer(yyjson_doc *doc, - const char *ptr); +/** Invalid input parameter, such as NULL input. */ +static const yyjson_ptr_code YYJSON_PTR_ERR_PARAMETER = 1; + +/** JSON pointer syntax error, such as invalid escape, token no prefix. */ +static const yyjson_ptr_code YYJSON_PTR_ERR_SYNTAX = 2; + +/** JSON pointer resolve failed, such as index out of range, key not found. */ +static const yyjson_ptr_code YYJSON_PTR_ERR_RESOLVE = 3; + +/** Document's root is NULL, but it is required for the function call. */ +static const yyjson_ptr_code YYJSON_PTR_ERR_NULL_ROOT = 4; -/** Get a JSON value with JSON Pointer (RFC 6901). - The `ptr` should be a UTF-8 string, null-terminator is not required. - The `len` should be the length of the `ptr`, in bytes. +/** Cannot set root as the target is not a document. */ +static const yyjson_ptr_code YYJSON_PTR_ERR_SET_ROOT = 5; + +/** The memory allocation failed and a new value could not be created. */ +static const yyjson_ptr_code YYJSON_PTR_ERR_MEMORY_ALLOCATION = 6; + +/** Error information for JSON pointer. */ +typedef struct yyjson_ptr_err { + /** Error code, see `yyjson_ptr_code` for all possible values. */ + yyjson_ptr_code code; + /** Error message, constant, no need to free (NULL if no error). */ + const char *msg; + /** Error byte position for input JSON pointer (0 if no error). */ + size_t pos; +} yyjson_ptr_err; + +/** + A context for JSON pointer operation. - Returns NULL if there's no matched value. */ -yyjson_api_inline yyjson_val *yyjson_doc_get_pointern(yyjson_doc *doc, - const char *ptr, - size_t len); + This struct stores the context of JSON Pointer operation result. The struct + can be used with three helper functions: `ctx_append()`, `ctx_replace()`, and + `ctx_remove()`, which perform the corresponding operations on the container + without re-parsing the JSON Pointer. + + For example: + @code + // doc before: {"a":[0,1,null]} + // ptr: "/a/2" + val = yyjson_mut_doc_ptr_getx(doc, ptr, strlen(ptr), &ctx, &err); + if (yyjson_is_null(val)) { + yyjson_ptr_ctx_remove(&ctx); + } + // doc after: {"a":[0,1]} + @endcode + */ +typedef struct yyjson_ptr_ctx { + /** + The container (parent) of the target value. It can be either an array or + an object. If the target location has no value, but all its parent + containers exist, and the target location can be used to insert a new + value, then `ctn` is the parent container of the target location. + Otherwise, `ctn` is NULL. + */ + yyjson_mut_val *ctn; + /** + The previous sibling of the target value. It can be either a value in an + array or a key in an object. As the container is a `circular linked list` + of elements, `pre` is the previous node of the target value. If the + operation is `add` or `set`, then `pre` is the previous node of the new + value, not the original target value. If the target value does not exist, + `pre` is NULL. + */ + yyjson_mut_val *pre; + /** + The removed value if the operation is `set`, `replace` or `remove`. It can + be used to restore the original state of the document if needed. + */ + yyjson_mut_val *old; +} yyjson_ptr_ctx; -/** Get a JSON value with JSON Pointer (RFC 6901). - The `ptr` should be a null-terminated UTF-8 string. - - Returns NULL if there's no matched value. */ -yyjson_api_inline yyjson_mut_val *yyjson_mut_get_pointer(yyjson_mut_val *val, +/** + Get value by a JSON Pointer. + @param doc The JSON document to be queried. + @param ptr The JSON pointer string (UTF-8 with null-terminator). + @return The value referenced by the JSON pointer. + NULL if `doc` or `ptr` is NULL, or the JSON pointer cannot be resolved. + */ +yyjson_api_inline yyjson_val *yyjson_doc_ptr_get(yyjson_doc *doc, + const char *ptr); + +/** + Get value by a JSON Pointer. + @param doc The JSON document to be queried. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @return The value referenced by the JSON pointer. + NULL if `doc` or `ptr` is NULL, or the JSON pointer cannot be resolved. + */ +yyjson_api_inline yyjson_val *yyjson_doc_ptr_getn(yyjson_doc *doc, + const char *ptr, size_t len); + +/** + Get value by a JSON Pointer. + @param doc The JSON document to be queried. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @param err A pointer to store the error information, or NULL if not needed. + @return The value referenced by the JSON pointer. + NULL if `doc` or `ptr` is NULL, or the JSON pointer cannot be resolved. + */ +yyjson_api_inline yyjson_val *yyjson_doc_ptr_getx(yyjson_doc *doc, + const char *ptr, size_t len, + yyjson_ptr_err *err); + +/** + Get value by a JSON Pointer. + @param val The JSON value to be queried. + @param ptr The JSON pointer string (UTF-8 with null-terminator). + @return The value referenced by the JSON pointer. + NULL if `val` or `ptr` is NULL, or the JSON pointer cannot be resolved. + */ +yyjson_api_inline yyjson_val *yyjson_ptr_get(yyjson_val *val, + const char *ptr); + +/** + Get value by a JSON Pointer. + @param val The JSON value to be queried. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @return The value referenced by the JSON pointer. + NULL if `val` or `ptr` is NULL, or the JSON pointer cannot be resolved. + */ +yyjson_api_inline yyjson_val *yyjson_ptr_getn(yyjson_val *val, + const char *ptr, size_t len); + +/** + Get value by a JSON Pointer. + @param val The JSON value to be queried. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @param err A pointer to store the error information, or NULL if not needed. + @return The value referenced by the JSON pointer. + NULL if `val` or `ptr` is NULL, or the JSON pointer cannot be resolved. + */ +yyjson_api_inline yyjson_val *yyjson_ptr_getx(yyjson_val *val, + const char *ptr, size_t len, + yyjson_ptr_err *err); + +/** + Get value by a JSON Pointer. + @param doc The JSON document to be queried. + @param ptr The JSON pointer string (UTF-8 with null-terminator). + @return The value referenced by the JSON pointer. + NULL if `doc` or `ptr` is NULL, or the JSON pointer cannot be resolved. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_ptr_get(yyjson_mut_doc *doc, const char *ptr); -/** Get a JSON value with JSON Pointer (RFC 6901). - The `ptr` should be a UTF-8 string, null-terminator is not required. - The `len` should be the length of the `ptr`, in bytes. - - Returns NULL if there's no matched value. */ -yyjson_api_inline yyjson_mut_val *yyjson_mut_get_pointern(yyjson_mut_val *val, +/** + Get value by a JSON Pointer. + @param doc The JSON document to be queried. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @return The value referenced by the JSON pointer. + NULL if `doc` or `ptr` is NULL, or the JSON pointer cannot be resolved. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_ptr_getn(yyjson_mut_doc *doc, const char *ptr, size_t len); -/** Get a JSON value with JSON Pointer (RFC 6901). - The `ptr` should be a null-terminated UTF-8 string. - - Returns NULL if there's no matched value. */ -yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_get_pointer( +/** + Get value by a JSON Pointer. + @param doc The JSON document to be queried. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @param ctx A pointer to store the result context, or NULL if not needed. + @param err A pointer to store the error information, or NULL if not needed. + @return The value referenced by the JSON pointer. + NULL if `doc` or `ptr` is NULL, or the JSON pointer cannot be resolved. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_ptr_getx(yyjson_mut_doc *doc, + const char *ptr, + size_t len, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err); + +/** + Get value by a JSON Pointer. + @param val The JSON value to be queried. + @param ptr The JSON pointer string (UTF-8 with null-terminator). + @return The value referenced by the JSON pointer. + NULL if `val` or `ptr` is NULL, or the JSON pointer cannot be resolved. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_ptr_get(yyjson_mut_val *val, + const char *ptr); + +/** + Get value by a JSON Pointer. + @param val The JSON value to be queried. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @return The value referenced by the JSON pointer. + NULL if `val` or `ptr` is NULL, or the JSON pointer cannot be resolved. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_ptr_getn(yyjson_mut_val *val, + const char *ptr, + size_t len); + +/** + Get value by a JSON Pointer. + @param val The JSON value to be queried. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @param ctx A pointer to store the result context, or NULL if not needed. + @param err A pointer to store the error information, or NULL if not needed. + @return The value referenced by the JSON pointer. + NULL if `val` or `ptr` is NULL, or the JSON pointer cannot be resolved. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_ptr_getx(yyjson_mut_val *val, + const char *ptr, + size_t len, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err); + +/** + Add (insert) value by a JSON pointer. + @param doc The target JSON document. + @param ptr The JSON pointer string (UTF-8 with null-terminator). + @param new_val The value to be added. + @return true if JSON pointer is valid and new value is added, false otherwise. + @note The parent nodes will be created if they do not exist. + */ +yyjson_api_inline bool yyjson_mut_doc_ptr_add(yyjson_mut_doc *doc, + const char *ptr, + yyjson_mut_val *new_val); + +/** + Add (insert) value by a JSON pointer. + @param doc The target JSON document. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @param new_val The value to be added. + @return true if JSON pointer is valid and new value is added, false otherwise. + @note The parent nodes will be created if they do not exist. + */ +yyjson_api_inline bool yyjson_mut_doc_ptr_addn(yyjson_mut_doc *doc, + const char *ptr, size_t len, + yyjson_mut_val *new_val); + +/** + Add (insert) value by a JSON pointer. + @param doc The target JSON document. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @param new_val The value to be added. + @param create_parent Whether to create parent nodes if not exist. + @param ctx A pointer to store the result context, or NULL if not needed. + @param err A pointer to store the error information, or NULL if not needed. + @return true if JSON pointer is valid and new value is added, false otherwise. + */ +yyjson_api_inline bool yyjson_mut_doc_ptr_addx(yyjson_mut_doc *doc, + const char *ptr, size_t len, + yyjson_mut_val *new_val, + bool create_parent, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err); + +/** + Add (insert) value by a JSON pointer. + @param val The target JSON value. + @param ptr The JSON pointer string (UTF-8 with null-terminator). + @param doc Only used to create new values when needed. + @param new_val The value to be added. + @return true if JSON pointer is valid and new value is added, false otherwise. + @note The parent nodes will be created if they do not exist. + */ +yyjson_api_inline bool yyjson_mut_ptr_add(yyjson_mut_val *val, + const char *ptr, + yyjson_mut_val *new_val, + yyjson_mut_doc *doc); + +/** + Add (insert) value by a JSON pointer. + @param val The target JSON value. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @param doc Only used to create new values when needed. + @param new_val The value to be added. + @return true if JSON pointer is valid and new value is added, false otherwise. + @note The parent nodes will be created if they do not exist. + */ +yyjson_api_inline bool yyjson_mut_ptr_addn(yyjson_mut_val *val, + const char *ptr, size_t len, + yyjson_mut_val *new_val, + yyjson_mut_doc *doc); + +/** + Add (insert) value by a JSON pointer. + @param val The target JSON value. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @param doc Only used to create new values when needed. + @param new_val The value to be added. + @param create_parent Whether to create parent nodes if not exist. + @param ctx A pointer to store the result context, or NULL if not needed. + @param err A pointer to store the error information, or NULL if not needed. + @return true if JSON pointer is valid and new value is added, false otherwise. + */ +yyjson_api_inline bool yyjson_mut_ptr_addx(yyjson_mut_val *val, + const char *ptr, size_t len, + yyjson_mut_val *new_val, + yyjson_mut_doc *doc, + bool create_parent, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err); + +/** + Set value by a JSON pointer. + @param doc The target JSON document. + @param ptr The JSON pointer string (UTF-8 with null-terminator). + @param new_val The value to be set, pass NULL to remove. + @return true if JSON pointer is valid and new value is set, false otherwise. + @note The parent nodes will be created if they do not exist. + If the target value already exists, it will be replaced by the new value. + */ +yyjson_api_inline bool yyjson_mut_doc_ptr_set(yyjson_mut_doc *doc, + const char *ptr, + yyjson_mut_val *new_val); + +/** + Set value by a JSON pointer. + @param doc The target JSON document. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @param new_val The value to be set, pass NULL to remove. + @return true if JSON pointer is valid and new value is set, false otherwise. + @note The parent nodes will be created if they do not exist. + If the target value already exists, it will be replaced by the new value. + */ +yyjson_api_inline bool yyjson_mut_doc_ptr_setn(yyjson_mut_doc *doc, + const char *ptr, size_t len, + yyjson_mut_val *new_val); + +/** + Set value by a JSON pointer. + @param doc The target JSON document. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @param new_val The value to be set, pass NULL to remove. + @param create_parent Whether to create parent nodes if not exist. + @param ctx A pointer to store the result context, or NULL if not needed. + @param err A pointer to store the error information, or NULL if not needed. + @return true if JSON pointer is valid and new value is set, false otherwise. + @note If the target value already exists, it will be replaced by the new value. + */ +yyjson_api_inline bool yyjson_mut_doc_ptr_setx(yyjson_mut_doc *doc, + const char *ptr, size_t len, + yyjson_mut_val *new_val, + bool create_parent, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err); + +/** + Set value by a JSON pointer. + @param val The target JSON value. + @param ptr The JSON pointer string (UTF-8 with null-terminator). + @param new_val The value to be set, pass NULL to remove. + @param doc Only used to create new values when needed. + @return true if JSON pointer is valid and new value is set, false otherwise. + @note The parent nodes will be created if they do not exist. + If the target value already exists, it will be replaced by the new value. + */ +yyjson_api_inline bool yyjson_mut_ptr_set(yyjson_mut_val *val, + const char *ptr, + yyjson_mut_val *new_val, + yyjson_mut_doc *doc); + +/** + Set value by a JSON pointer. + @param val The target JSON value. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @param new_val The value to be set, pass NULL to remove. + @param doc Only used to create new values when needed. + @return true if JSON pointer is valid and new value is set, false otherwise. + @note The parent nodes will be created if they do not exist. + If the target value already exists, it will be replaced by the new value. + */ +yyjson_api_inline bool yyjson_mut_ptr_setn(yyjson_mut_val *val, + const char *ptr, size_t len, + yyjson_mut_val *new_val, + yyjson_mut_doc *doc); + +/** + Set value by a JSON pointer. + @param val The target JSON value. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @param new_val The value to be set, pass NULL to remove. + @param doc Only used to create new values when needed. + @param create_parent Whether to create parent nodes if not exist. + @param ctx A pointer to store the result context, or NULL if not needed. + @param err A pointer to store the error information, or NULL if not needed. + @return true if JSON pointer is valid and new value is set, false otherwise. + @note If the target value already exists, it will be replaced by the new value. + */ +yyjson_api_inline bool yyjson_mut_ptr_setx(yyjson_mut_val *val, + const char *ptr, size_t len, + yyjson_mut_val *new_val, + yyjson_mut_doc *doc, + bool create_parent, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err); + +/** + Replace value by a JSON pointer. + @param doc The target JSON document. + @param ptr The JSON pointer string (UTF-8 with null-terminator). + @param new_val The new value to replace the old one. + @return The old value that was replaced, or NULL if not found. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_ptr_replace( + yyjson_mut_doc *doc, const char *ptr, yyjson_mut_val *new_val); + +/** + Replace value by a JSON pointer. + @param doc The target JSON document. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @param new_val The new value to replace the old one. + @return The old value that was replaced, or NULL if not found. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_ptr_replacen( + yyjson_mut_doc *doc, const char *ptr, size_t len, yyjson_mut_val *new_val); + +/** + Replace value by a JSON pointer. + @param doc The target JSON document. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @param new_val The new value to replace the old one. + @param ctx A pointer to store the result context, or NULL if not needed. + @param err A pointer to store the error information, or NULL if not needed. + @return The old value that was replaced, or NULL if not found. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_ptr_replacex( + yyjson_mut_doc *doc, const char *ptr, size_t len, yyjson_mut_val *new_val, + yyjson_ptr_ctx *ctx, yyjson_ptr_err *err); + +/** + Replace value by a JSON pointer. + @param val The target JSON value. + @param ptr The JSON pointer string (UTF-8 with null-terminator). + @param new_val The new value to replace the old one. + @return The old value that was replaced, or NULL if not found. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_ptr_replace( + yyjson_mut_val *val, const char *ptr, yyjson_mut_val *new_val); + +/** + Replace value by a JSON pointer. + @param val The target JSON value. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @param new_val The new value to replace the old one. + @return The old value that was replaced, or NULL if not found. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_ptr_replacen( + yyjson_mut_val *val, const char *ptr, size_t len, yyjson_mut_val *new_val); + +/** + Replace value by a JSON pointer. + @param val The target JSON value. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @param new_val The new value to replace the old one. + @param ctx A pointer to store the result context, or NULL if not needed. + @param err A pointer to store the error information, or NULL if not needed. + @return The old value that was replaced, or NULL if not found. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_ptr_replacex( + yyjson_mut_val *val, const char *ptr, size_t len, yyjson_mut_val *new_val, + yyjson_ptr_ctx *ctx, yyjson_ptr_err *err); + +/** + Remove value by a JSON pointer. + @param doc The target JSON document. + @param ptr The JSON pointer string (UTF-8 with null-terminator). + @return The removed value, or NULL on error. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_ptr_remove( yyjson_mut_doc *doc, const char *ptr); -/** Get a JSON value with JSON Pointer (RFC 6901). - The `ptr` should be a UTF-8 string, null-terminator is not required. - The `len` should be the length of the `ptr`, in bytes. - - Returns NULL if there's no matched value. */ -yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_get_pointern( +/** + Remove value by a JSON pointer. + @param doc The target JSON document. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @return The removed value, or NULL on error. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_ptr_removen( yyjson_mut_doc *doc, const char *ptr, size_t len); +/** + Remove value by a JSON pointer. + @param doc The target JSON document. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @param ctx A pointer to store the result context, or NULL if not needed. + @param err A pointer to store the error information, or NULL if not needed. + @return The removed value, or NULL on error. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_ptr_removex( + yyjson_mut_doc *doc, const char *ptr, size_t len, + yyjson_ptr_ctx *ctx, yyjson_ptr_err *err); + +/** + Remove value by a JSON pointer. + @param val The target JSON value. + @param ptr The JSON pointer string (UTF-8 with null-terminator). + @return The removed value, or NULL on error. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_ptr_remove(yyjson_mut_val *val, + const char *ptr); + +/** + Remove value by a JSON pointer. + @param val The target JSON value. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @return The removed value, or NULL on error. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_ptr_removen(yyjson_mut_val *val, + const char *ptr, + size_t len); + +/** + Remove value by a JSON pointer. + @param val The target JSON value. + @param ptr The JSON pointer string (UTF-8, null-terminator is not required). + @param len The length of `ptr` in bytes. + @param ctx A pointer to store the result context, or NULL if not needed. + @param err A pointer to store the error information, or NULL if not needed. + @return The removed value, or NULL on error. + */ +yyjson_api_inline yyjson_mut_val *yyjson_mut_ptr_removex(yyjson_mut_val *val, + const char *ptr, + size_t len, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err); + +/** + Append value by JSON pointer context. + @param ctx The context from the `yyjson_mut_ptr_xxx()` calls. + @param key New key if `ctx->ctn` is object, or NULL if `ctx->ctn` is array. + @param val New value to be added. + @return true on success or false on fail. + */ +yyjson_api_inline bool yyjson_ptr_ctx_append(yyjson_ptr_ctx *ctx, + yyjson_mut_val *key, + yyjson_mut_val *val); + +/** + Replace value by JSON pointer context. + @param ctx The context from the `yyjson_mut_ptr_xxx()` calls. + @param val New value to be replaced. + @return true on success or false on fail. + @note If success, the old value will be returned via `ctx->old`. + */ +yyjson_api_inline bool yyjson_ptr_ctx_replace(yyjson_ptr_ctx *ctx, + yyjson_mut_val *val); + +/** + Remove value by JSON pointer context. + @param ctx The context from the `yyjson_mut_ptr_xxx()` calls. + @return true on success or false on fail. + @note If success, the old value will be returned via `ctx->old`. + */ +yyjson_api_inline bool yyjson_ptr_ctx_remove(yyjson_ptr_ctx *ctx); + + + +/*============================================================================== + * JSON Patch API (RFC 6902) + * https://tools.ietf.org/html/rfc6902 + *============================================================================*/ + +/** Result code for JSON patch. */ +typedef uint32_t yyjson_patch_code; + +/** Success, no error. */ +static const yyjson_patch_code YYJSON_PATCH_SUCCESS = 0; + +/** Invalid parameter, such as NULL input or non-array patch. */ +static const yyjson_patch_code YYJSON_PATCH_ERROR_INVALID_PARAMETER = 1; + +/** Memory allocation failure occurs. */ +static const yyjson_patch_code YYJSON_PATCH_ERROR_MEMORY_ALLOCATION = 2; + +/** JSON patch operation is not object type. */ +static const yyjson_patch_code YYJSON_PATCH_ERROR_INVALID_OPERATION = 3; + +/** JSON patch operation is missing a required key. */ +static const yyjson_patch_code YYJSON_PATCH_ERROR_MISSING_KEY = 4; + +/** JSON patch operation member is invalid. */ +static const yyjson_patch_code YYJSON_PATCH_ERROR_INVALID_MEMBER = 5; + +/** JSON patch operation `test` not equal. */ +static const yyjson_patch_code YYJSON_PATCH_ERROR_EQUAL = 6; + +/** JSON patch operation failed on JSON pointer. */ +static const yyjson_patch_code YYJSON_PATCH_ERROR_POINTER = 7; + +/** Error information for JSON patch. */ +typedef struct yyjson_patch_err { + /** Error code, see `yyjson_patch_code` for all possible values. */ + yyjson_patch_code code; + /** Index of the error operation (0 if no error). */ + size_t idx; + /** Error message, constant, no need to free (NULL if no error). */ + const char *msg; + /** JSON pointer error if `code == YYJSON_PATCH_ERROR_POINTER`. */ + yyjson_ptr_err ptr; +} yyjson_patch_err; + +/** + Creates and returns a patched JSON value (RFC 6902). + The memory of the returned value is allocated by the `doc`. + The `err` is used to receive error information, pass NULL if not needed. + Returns NULL if the patch could not be applied. + */ +yyjson_api yyjson_mut_val *yyjson_patch(yyjson_mut_doc *doc, + yyjson_val *orig, + yyjson_val *patch, + yyjson_patch_err *err); + +/** + Creates and returns a patched JSON value (RFC 6902). + The memory of the returned value is allocated by the `doc`. + The `err` is used to receive error information, pass NULL if not needed. + Returns NULL if the patch could not be applied. + */ +yyjson_api yyjson_mut_val *yyjson_mut_patch(yyjson_mut_doc *doc, + yyjson_mut_val *orig, + yyjson_mut_val *patch, + yyjson_patch_err *err); + /*============================================================================== - * JSON Merge-Patch API + * JSON Merge-Patch API (RFC 7386) * https://tools.ietf.org/html/rfc7386 *============================================================================*/ -/** Creates and returns a merge-patched JSON value (RFC 7386). - The memory of the returned value is allocated by the `doc`. - Returns NULL if the patch could not be applied. - - @warning This function is recursive and may cause a stack overflow if the - object level is too deep. */ +/** + Creates and returns a merge-patched JSON value (RFC 7386). + The memory of the returned value is allocated by the `doc`. + Returns NULL if the patch could not be applied. + + @warning This function is recursive and may cause a stack overflow if the + object level is too deep. + */ yyjson_api yyjson_mut_val *yyjson_merge_patch(yyjson_mut_doc *doc, yyjson_val *orig, yyjson_val *patch); -/** Creates and returns a merge-patched JSON value (RFC 7386). - The memory of the returned value is allocated by the `doc`. - Returns NULL if the patch could not be applied. - - @warning This function is recursive and may cause a stack overflow if the - object level is too deep. */ +/** + Creates and returns a merge-patched JSON value (RFC 7386). + The memory of the returned value is allocated by the `doc`. + Returns NULL if the patch could not be applied. + + @warning This function is recursive and may cause a stack overflow if the + object level is too deep. + */ yyjson_api yyjson_mut_val *yyjson_mut_merge_patch(yyjson_mut_doc *doc, yyjson_mut_val *orig, yyjson_mut_val *patch); @@ -3290,6 +4517,58 @@ struct yyjson_doc { * Unsafe JSON Value API (Implementation) *============================================================================*/ +/* + Whether the string does not need to be escaped for serialization. + This function is used to optimize the writing speed of small constant strings. + This function works only if the compiler can evaluate it at compile time. + + Clang supports it since v8.0, + earlier versions do not support constant_p(strlen) and return false. + GCC supports it since at least v4.4, + earlier versions may compile it as run-time instructions. + ICC supports it since at least v16, + earlier versions are uncertain. + + @param str The C string. + @param len The returnd value from strlen(str). + */ +yyjson_api_inline bool unsafe_yyjson_is_str_noesc(const char *str, size_t len) { +#if YYJSON_HAS_CONSTANT_P && \ + (!YYJSON_IS_REAL_GCC || yyjson_gcc_available(4, 4, 0)) + if (yyjson_constant_p(len) && len <= 32) { + /* + Same as the following loop: + + for (size_t i = 0; i < len; i++) { + char c = str[i]; + if (c < ' ' || c > '~' || c == '"' || c == '\\') return false; + } + + GCC evaluates it at compile time only if the string length is within 17 + and -O3 (which turns on the -fpeel-loops flag) is used. + So the loop is unrolled for GCC. + */ +# define yyjson_repeat32_incr(x) \ + x(0) x(1) x(2) x(3) x(4) x(5) x(6) x(7) \ + x(8) x(9) x(10) x(11) x(12) x(13) x(14) x(15) \ + x(16) x(17) x(18) x(19) x(20) x(21) x(22) x(23) \ + x(24) x(25) x(26) x(27) x(28) x(29) x(30) x(31) +# define yyjson_check_char_noesc(i) \ + if (i < len) { \ + char c = str[i]; \ + if (c < ' ' || c > '~' || c == '"' || c == '\\') return false; } + yyjson_repeat32_incr(yyjson_check_char_noesc) +# undef yyjson_repeat32_incr +# undef yyjson_check_char_noesc + return true; + } +#else + (void)str; + (void)len; +#endif + return false; +} + yyjson_api_inline yyjson_type unsafe_yyjson_get_type(void *val) { uint8_t tag = (uint8_t)((yyjson_val *)val)->tag; return (yyjson_type)(tag & YYJSON_TYPE_MASK); @@ -3400,6 +4679,28 @@ yyjson_api_inline double unsafe_yyjson_get_real(void *val) { return ((yyjson_val *)val)->uni.f64; } +yyjson_api_inline double unsafe_yyjson_get_num(void *val) { + uint8_t tag = unsafe_yyjson_get_tag(val); + if (tag == (YYJSON_TYPE_NUM | YYJSON_SUBTYPE_REAL)) { + return ((yyjson_val *)val)->uni.f64; + } else if (tag == (YYJSON_TYPE_NUM | YYJSON_SUBTYPE_SINT)) { + return (double)((yyjson_val *)val)->uni.i64; + } else if (tag == (YYJSON_TYPE_NUM | YYJSON_SUBTYPE_UINT)) { +#if YYJSON_U64_TO_F64_NO_IMPL + uint64_t msb = ((uint64_t)1) << 63; + uint64_t num = ((yyjson_val *)val)->uni.u64; + if ((num & msb) == 0) { + return (double)(int64_t)num; + } else { + return ((double)(int64_t)((num >> 1) | (num & 1))) * (double)2.0; + } +#else + return (double)((yyjson_val *)val)->uni.u64; +#endif + } + return 0.0; +} + yyjson_api_inline const char *unsafe_yyjson_get_str(void *val) { return ((yyjson_val *)val)->uni.str; } @@ -3408,12 +4709,6 @@ yyjson_api_inline size_t unsafe_yyjson_get_len(void *val) { return (size_t)(((yyjson_val *)val)->tag >> YYJSON_TAG_BIT); } -yyjson_api_inline void unsafe_yyjson_set_len(void *val, size_t len) { - uint64_t tag = ((yyjson_val *)val)->tag & YYJSON_TAG_MASK; - tag |= (uint64_t)len << YYJSON_TAG_BIT; - ((yyjson_val *)val)->tag = tag; -} - yyjson_api_inline yyjson_val *unsafe_yyjson_get_first(yyjson_val *ctn) { return ctn + 1; } @@ -3427,15 +4722,96 @@ yyjson_api_inline yyjson_val *unsafe_yyjson_get_next(yyjson_val *val) { yyjson_api_inline bool unsafe_yyjson_equals_strn(void *val, const char *str, size_t len) { - uint64_t tag = ((uint64_t)len << YYJSON_TAG_BIT) | YYJSON_TYPE_STR; - return ((yyjson_val *)val)->tag == tag && - memcmp(((yyjson_val *)val)->uni.str, str, len) == 0; + return unsafe_yyjson_get_len(val) == len && + memcmp(((yyjson_val *)val)->uni.str, str, len) == 0; } yyjson_api_inline bool unsafe_yyjson_equals_str(void *val, const char *str) { return unsafe_yyjson_equals_strn(val, str, strlen(str)); } +yyjson_api_inline void unsafe_yyjson_set_type(void *val, yyjson_type type, + yyjson_subtype subtype) { + uint8_t tag = (type | subtype); + uint64_t new_tag = ((yyjson_val *)val)->tag; + new_tag = (new_tag & (~(uint64_t)YYJSON_TAG_MASK)) | (uint64_t)tag; + ((yyjson_val *)val)->tag = new_tag; +} + +yyjson_api_inline void unsafe_yyjson_set_len(void *val, size_t len) { + uint64_t tag = ((yyjson_val *)val)->tag & YYJSON_TAG_MASK; + tag |= (uint64_t)len << YYJSON_TAG_BIT; + ((yyjson_val *)val)->tag = tag; +} + +yyjson_api_inline void unsafe_yyjson_inc_len(void *val) { + uint64_t tag = ((yyjson_val *)val)->tag; + tag += (uint64_t)(1 << YYJSON_TAG_BIT); + ((yyjson_val *)val)->tag = tag; +} + +yyjson_api_inline void unsafe_yyjson_set_raw(void *val, const char *raw, + size_t len) { + unsafe_yyjson_set_type(val, YYJSON_TYPE_RAW, YYJSON_SUBTYPE_NONE); + unsafe_yyjson_set_len(val, len); + ((yyjson_val *)val)->uni.str = raw; +} + +yyjson_api_inline void unsafe_yyjson_set_null(void *val) { + unsafe_yyjson_set_type(val, YYJSON_TYPE_NULL, YYJSON_SUBTYPE_NONE); + unsafe_yyjson_set_len(val, 0); +} + +yyjson_api_inline void unsafe_yyjson_set_bool(void *val, bool num) { + yyjson_subtype subtype = num ? YYJSON_SUBTYPE_TRUE : YYJSON_SUBTYPE_FALSE; + unsafe_yyjson_set_type(val, YYJSON_TYPE_BOOL, subtype); + unsafe_yyjson_set_len(val, 0); +} + +yyjson_api_inline void unsafe_yyjson_set_uint(void *val, uint64_t num) { + unsafe_yyjson_set_type(val, YYJSON_TYPE_NUM, YYJSON_SUBTYPE_UINT); + unsafe_yyjson_set_len(val, 0); + ((yyjson_val *)val)->uni.u64 = num; +} + +yyjson_api_inline void unsafe_yyjson_set_sint(void *val, int64_t num) { + unsafe_yyjson_set_type(val, YYJSON_TYPE_NUM, YYJSON_SUBTYPE_SINT); + unsafe_yyjson_set_len(val, 0); + ((yyjson_val *)val)->uni.i64 = num; +} + +yyjson_api_inline void unsafe_yyjson_set_real(void *val, double num) { + unsafe_yyjson_set_type(val, YYJSON_TYPE_NUM, YYJSON_SUBTYPE_REAL); + unsafe_yyjson_set_len(val, 0); + ((yyjson_val *)val)->uni.f64 = num; +} + +yyjson_api_inline void unsafe_yyjson_set_str(void *val, const char *str) { + size_t len = strlen(str); + bool noesc = unsafe_yyjson_is_str_noesc(str, len); + yyjson_subtype sub = noesc ? YYJSON_SUBTYPE_NOESC : YYJSON_SUBTYPE_NONE; + unsafe_yyjson_set_type(val, YYJSON_TYPE_STR, sub); + unsafe_yyjson_set_len(val, len); + ((yyjson_val *)val)->uni.str = str; +} + +yyjson_api_inline void unsafe_yyjson_set_strn(void *val, const char *str, + size_t len) { + unsafe_yyjson_set_type(val, YYJSON_TYPE_STR, YYJSON_SUBTYPE_NONE); + unsafe_yyjson_set_len(val, len); + ((yyjson_val *)val)->uni.str = str; +} + +yyjson_api_inline void unsafe_yyjson_set_arr(void *val, size_t size) { + unsafe_yyjson_set_type(val, YYJSON_TYPE_ARR, YYJSON_SUBTYPE_NONE); + unsafe_yyjson_set_len(val, size); +} + +yyjson_api_inline void unsafe_yyjson_set_obj(void *val, size_t size) { + unsafe_yyjson_set_type(val, YYJSON_TYPE_OBJ, YYJSON_SUBTYPE_NONE); + unsafe_yyjson_set_len(val, size); +} + /*============================================================================== @@ -3457,6 +4833,7 @@ yyjson_api_inline size_t yyjson_doc_get_val_count(yyjson_doc *doc) { yyjson_api void yyjson_doc_free(yyjson_doc *doc) { if (doc) { yyjson_alc alc = doc->alc; + memset(&doc->alc, 0, sizeof(alc)); if (doc->str_pool) alc.free(alc.ctx, doc->str_pool); alc.free(alc.ctx, doc); } @@ -3547,6 +4924,7 @@ yyjson_api_inline const char *yyjson_get_type_desc(yyjson_val *val) { case YYJSON_TYPE_RAW | YYJSON_SUBTYPE_NONE: return "raw"; case YYJSON_TYPE_NULL | YYJSON_SUBTYPE_NONE: return "null"; case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NONE: return "string"; + case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NOESC: return "string"; case YYJSON_TYPE_ARR | YYJSON_SUBTYPE_NONE: return "array"; case YYJSON_TYPE_OBJ | YYJSON_SUBTYPE_NONE: return "object"; case YYJSON_TYPE_BOOL | YYJSON_SUBTYPE_TRUE: return "true"; @@ -3582,6 +4960,10 @@ yyjson_api_inline double yyjson_get_real(yyjson_val *val) { return yyjson_is_real(val) ? unsafe_yyjson_get_real(val) : 0.0; } +yyjson_api_inline double yyjson_get_num(yyjson_val *val) { + return val ? unsafe_yyjson_get_num(val) : 0.0; +} + yyjson_api_inline const char *yyjson_get_str(yyjson_val *val) { return yyjson_is_str(val) ? unsafe_yyjson_get_str(val) : NULL; } @@ -3592,7 +4974,8 @@ yyjson_api_inline size_t yyjson_get_len(yyjson_val *val) { yyjson_api_inline bool yyjson_equals_str(yyjson_val *val, const char *str) { if (yyjson_likely(val && str)) { - return unsafe_yyjson_equals_str(val, str); + return unsafe_yyjson_is_str(val) && + unsafe_yyjson_equals_str(val, str); } return false; } @@ -3600,7 +4983,8 @@ yyjson_api_inline bool yyjson_equals_str(yyjson_val *val, const char *str) { yyjson_api_inline bool yyjson_equals_strn(yyjson_val *val, const char *str, size_t len) { if (yyjson_likely(val && str)) { - return unsafe_yyjson_equals_strn(val, str, len); + return unsafe_yyjson_is_str(val) && + unsafe_yyjson_equals_strn(val, str, len); } return false; } @@ -3608,12 +4992,68 @@ yyjson_api_inline bool yyjson_equals_strn(yyjson_val *val, const char *str, yyjson_api bool unsafe_yyjson_equals(yyjson_val *lhs, yyjson_val *rhs); yyjson_api_inline bool yyjson_equals(yyjson_val *lhs, yyjson_val *rhs) { - if (yyjson_unlikely(!lhs || !rhs)) - return false; - + if (yyjson_unlikely(!lhs || !rhs)) return false; return unsafe_yyjson_equals(lhs, rhs); } +yyjson_api_inline bool yyjson_set_raw(yyjson_val *val, + const char *raw, size_t len) { + if (yyjson_unlikely(!val || unsafe_yyjson_is_ctn(val))) return false; + unsafe_yyjson_set_raw(val, raw, len); + return true; +} + +yyjson_api_inline bool yyjson_set_null(yyjson_val *val) { + if (yyjson_unlikely(!val || unsafe_yyjson_is_ctn(val))) return false; + unsafe_yyjson_set_null(val); + return true; +} + +yyjson_api_inline bool yyjson_set_bool(yyjson_val *val, bool num) { + if (yyjson_unlikely(!val || unsafe_yyjson_is_ctn(val))) return false; + unsafe_yyjson_set_bool(val, num); + return true; +} + +yyjson_api_inline bool yyjson_set_uint(yyjson_val *val, uint64_t num) { + if (yyjson_unlikely(!val || unsafe_yyjson_is_ctn(val))) return false; + unsafe_yyjson_set_uint(val, num); + return true; +} + +yyjson_api_inline bool yyjson_set_sint(yyjson_val *val, int64_t num) { + if (yyjson_unlikely(!val || unsafe_yyjson_is_ctn(val))) return false; + unsafe_yyjson_set_sint(val, num); + return true; +} + +yyjson_api_inline bool yyjson_set_int(yyjson_val *val, int num) { + if (yyjson_unlikely(!val || unsafe_yyjson_is_ctn(val))) return false; + unsafe_yyjson_set_sint(val, (int64_t)num); + return true; +} + +yyjson_api_inline bool yyjson_set_real(yyjson_val *val, double num) { + if (yyjson_unlikely(!val || unsafe_yyjson_is_ctn(val))) return false; + unsafe_yyjson_set_real(val, num); + return true; +} + +yyjson_api_inline bool yyjson_set_str(yyjson_val *val, const char *str) { + if (yyjson_unlikely(!val || unsafe_yyjson_is_ctn(val))) return false; + if (yyjson_unlikely(!str)) return false; + unsafe_yyjson_set_str(val, str); + return true; +} + +yyjson_api_inline bool yyjson_set_strn(yyjson_val *val, + const char *str, size_t len) { + if (yyjson_unlikely(!val || unsafe_yyjson_is_ctn(val))) return false; + if (yyjson_unlikely(!str)) return false; + unsafe_yyjson_set_strn(val, str, len); + return true; +} + /*============================================================================== @@ -3670,12 +5110,6 @@ yyjson_api_inline yyjson_val *yyjson_arr_get_last(yyjson_val *arr) { * JSON Array Iterator API (Implementation) *============================================================================*/ -struct yyjson_arr_iter { - size_t idx; /**< current index, from 0 */ - size_t max; /**< maximum index, idx < max */ - yyjson_val *cur; /**< current value */ -}; - yyjson_api_inline bool yyjson_arr_iter_init(yyjson_val *arr, yyjson_arr_iter *iter) { if (yyjson_likely(yyjson_is_arr(arr) && iter)) { @@ -3688,6 +5122,12 @@ yyjson_api_inline bool yyjson_arr_iter_init(yyjson_val *arr, return false; } +yyjson_api_inline yyjson_arr_iter yyjson_arr_iter_with(yyjson_val *arr) { + yyjson_arr_iter iter; + yyjson_arr_iter_init(arr, &iter); + return iter; +} + yyjson_api_inline bool yyjson_arr_iter_has_next(yyjson_arr_iter *iter) { return iter ? iter->idx < iter->max : false; } @@ -3721,15 +5161,11 @@ yyjson_api_inline yyjson_val *yyjson_obj_get(yyjson_val *obj, yyjson_api_inline yyjson_val *yyjson_obj_getn(yyjson_val *obj, const char *_key, size_t key_len) { - uint64_t tag = (((uint64_t)key_len) << YYJSON_TAG_BIT) | YYJSON_TYPE_STR; if (yyjson_likely(yyjson_is_obj(obj) && _key)) { size_t len = unsafe_yyjson_get_len(obj); yyjson_val *key = unsafe_yyjson_get_first(obj); while (len-- > 0) { - if (key->tag == tag && - memcmp(key->uni.ptr, _key, key_len) == 0) { - return key + 1; - } + if (unsafe_yyjson_equals_strn(key, _key, key_len)) return key + 1; key = unsafe_yyjson_get_next(key + 1); } } @@ -3742,13 +5178,6 @@ yyjson_api_inline yyjson_val *yyjson_obj_getn(yyjson_val *obj, * JSON Object Iterator API (Implementation) *============================================================================*/ -struct yyjson_obj_iter { - size_t idx; /**< current key index, from 0 */ - size_t max; /**< maximum key index, idx < max */ - yyjson_val *cur; /**< current key */ - yyjson_val *obj; /**< the object being iterated */ -}; - yyjson_api_inline bool yyjson_obj_iter_init(yyjson_val *obj, yyjson_obj_iter *iter) { if (yyjson_likely(yyjson_is_obj(obj) && iter)) { @@ -3762,6 +5191,12 @@ yyjson_api_inline bool yyjson_obj_iter_init(yyjson_val *obj, return false; } +yyjson_api_inline yyjson_obj_iter yyjson_obj_iter_with(yyjson_val *obj) { + yyjson_obj_iter iter; + yyjson_obj_iter_init(obj, &iter); + return iter; +} + yyjson_api_inline bool yyjson_obj_iter_has_next(yyjson_obj_iter *iter) { return iter ? iter->idx < iter->max : false; } @@ -3798,8 +5233,7 @@ yyjson_api_inline yyjson_val *yyjson_obj_iter_getn(yyjson_obj_iter *iter, } while (idx++ < max) { yyjson_val *next = unsafe_yyjson_get_next(cur + 1); - if (unsafe_yyjson_get_len(cur) == key_len && - memcmp(cur->uni.str, key, key_len) == 0) { + if (unsafe_yyjson_equals_strn(cur, key, key_len)) { iter->idx = idx; iter->cur = next; return cur + 1; @@ -3836,8 +5270,9 @@ struct yyjson_mut_val { A memory chunk in string memory pool. */ typedef struct yyjson_str_chunk { - struct yyjson_str_chunk *next; - /* flexible array member here */ + struct yyjson_str_chunk *next; /* next chunk linked list */ + size_t chunk_size; /* chunk size in bytes */ + /* char str[]; flexible array member */ } yyjson_str_chunk; /** @@ -3853,10 +5288,13 @@ typedef struct yyjson_str_pool { /** A memory chunk in value memory pool. + `sizeof(yyjson_val_chunk)` should not larger than `sizeof(yyjson_mut_val)`. */ typedef struct yyjson_val_chunk { - struct yyjson_val_chunk *next; - /* flexible array member here */ + struct yyjson_val_chunk *next; /* next chunk linked list */ + size_t chunk_size; /* chunk size in bytes */ + /* char pad[sizeof(yyjson_mut_val) - sizeof(yyjson_val_chunk)]; padding */ + /* yyjson_mut_val vals[]; flexible array member */ } yyjson_val_chunk; /** @@ -3879,27 +5317,34 @@ struct yyjson_mut_doc { /* Ensures the capacity to at least equal to the specified byte length. */ yyjson_api bool unsafe_yyjson_str_pool_grow(yyjson_str_pool *pool, - yyjson_alc *alc, size_t len); + const yyjson_alc *alc, + size_t len); /* Ensures the capacity to at least equal to the specified value count. */ yyjson_api bool unsafe_yyjson_val_pool_grow(yyjson_val_pool *pool, - yyjson_alc *alc, size_t count); + const yyjson_alc *alc, + size_t count); -yyjson_api_inline char *unsafe_yyjson_mut_strncpy(yyjson_mut_doc *doc, - const char *str, size_t len) { +/* Allocate memory for string. */ +yyjson_api_inline char *unsafe_yyjson_mut_str_alc(yyjson_mut_doc *doc, + size_t len) { char *mem; - yyjson_alc *alc = &doc->alc; + const yyjson_alc *alc = &doc->alc; yyjson_str_pool *pool = &doc->str_pool; - - if (!str) return NULL; if (yyjson_unlikely((size_t)(pool->end - pool->cur) <= len)) { if (yyjson_unlikely(!unsafe_yyjson_str_pool_grow(pool, alc, len + 1))) { return NULL; } } - mem = pool->cur; pool->cur = mem + len + 1; + return mem; +} + +yyjson_api_inline char *unsafe_yyjson_mut_strncpy(yyjson_mut_doc *doc, + const char *str, size_t len) { + char *mem = unsafe_yyjson_mut_str_alc(doc, len); + if (yyjson_unlikely(!mem)) return NULL; memcpy((void *)mem, (const void *)str, len); mem[len] = '\0'; return mem; @@ -3915,7 +5360,6 @@ yyjson_api_inline yyjson_mut_val *unsafe_yyjson_mut_val(yyjson_mut_doc *doc, return NULL; } } - val = pool->cur; pool->cur += count; return val; @@ -4044,6 +5488,10 @@ yyjson_api_inline double yyjson_mut_get_real(yyjson_mut_val *val) { return yyjson_get_real((yyjson_val *)val); } +yyjson_api_inline double yyjson_mut_get_num(yyjson_mut_val *val) { + return yyjson_get_num((yyjson_val *)val); +} + yyjson_api_inline const char *yyjson_mut_get_str(yyjson_mut_val *val) { return yyjson_get_str((yyjson_val *)val); } @@ -4071,6 +5519,75 @@ yyjson_api_inline bool yyjson_mut_equals(yyjson_mut_val *lhs, return unsafe_yyjson_mut_equals(lhs, rhs); } +yyjson_api_inline bool yyjson_mut_set_raw(yyjson_mut_val *val, + const char *raw, size_t len) { + if (yyjson_unlikely(!val || !raw)) return false; + unsafe_yyjson_set_raw(val, raw, len); + return true; +} + +yyjson_api_inline bool yyjson_mut_set_null(yyjson_mut_val *val) { + if (yyjson_unlikely(!val)) return false; + unsafe_yyjson_set_null(val); + return true; +} + +yyjson_api_inline bool yyjson_mut_set_bool(yyjson_mut_val *val, bool num) { + if (yyjson_unlikely(!val)) return false; + unsafe_yyjson_set_bool(val, num); + return true; +} + +yyjson_api_inline bool yyjson_mut_set_uint(yyjson_mut_val *val, uint64_t num) { + if (yyjson_unlikely(!val)) return false; + unsafe_yyjson_set_uint(val, num); + return true; +} + +yyjson_api_inline bool yyjson_mut_set_sint(yyjson_mut_val *val, int64_t num) { + if (yyjson_unlikely(!val)) return false; + unsafe_yyjson_set_sint(val, num); + return true; +} + +yyjson_api_inline bool yyjson_mut_set_int(yyjson_mut_val *val, int num) { + if (yyjson_unlikely(!val)) return false; + unsafe_yyjson_set_sint(val, (int64_t)num); + return true; +} + +yyjson_api_inline bool yyjson_mut_set_real(yyjson_mut_val *val, double num) { + if (yyjson_unlikely(!val)) return false; + unsafe_yyjson_set_real(val, num); + return true; +} + +yyjson_api_inline bool yyjson_mut_set_str(yyjson_mut_val *val, + const char *str) { + if (yyjson_unlikely(!val || !str)) return false; + unsafe_yyjson_set_str(val, str); + return true; +} + +yyjson_api_inline bool yyjson_mut_set_strn(yyjson_mut_val *val, + const char *str, size_t len) { + if (yyjson_unlikely(!val || !str)) return false; + unsafe_yyjson_set_strn(val, str, len); + return true; +} + +yyjson_api_inline bool yyjson_mut_set_arr(yyjson_mut_val *val) { + if (yyjson_unlikely(!val)) return false; + unsafe_yyjson_set_arr(val, 0); + return true; +} + +yyjson_api_inline bool yyjson_mut_set_obj(yyjson_mut_val *val) { + if (yyjson_unlikely(!val)) return false; + unsafe_yyjson_set_obj(val, 0); + return true; +} + /*============================================================================== @@ -4156,6 +5673,7 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_bool(yyjson_mut_doc *doc, if (yyjson_likely(doc)) { yyjson_mut_val *val = unsafe_yyjson_mut_val(doc, 1); if (yyjson_likely(val)) { + _val = !!_val; val->tag = YYJSON_TYPE_BOOL | (uint8_t)((uint8_t)_val << 3); return val; } @@ -4209,7 +5727,18 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_real(yyjson_mut_doc *doc, yyjson_api_inline yyjson_mut_val *yyjson_mut_str(yyjson_mut_doc *doc, const char *str) { - if (yyjson_likely(str)) return yyjson_mut_strn(doc, str, strlen(str)); + if (yyjson_likely(doc && str)) { + size_t len = strlen(str); + bool noesc = unsafe_yyjson_is_str_noesc(str, len); + yyjson_subtype sub = noesc ? YYJSON_SUBTYPE_NOESC : YYJSON_SUBTYPE_NONE; + yyjson_mut_val *val = unsafe_yyjson_mut_val(doc, 1); + if (yyjson_likely(val)) { + val->tag = ((uint64_t)len << YYJSON_TAG_BIT) | + (uint64_t)(YYJSON_TYPE_STR | sub); + val->uni.str = str; + return val; + } + } return NULL; } @@ -4229,7 +5758,19 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_strn(yyjson_mut_doc *doc, yyjson_api_inline yyjson_mut_val *yyjson_mut_strcpy(yyjson_mut_doc *doc, const char *str) { - if (yyjson_likely(str)) return yyjson_mut_strncpy(doc, str, strlen(str)); + if (yyjson_likely(doc && str)) { + size_t len = strlen(str); + bool noesc = unsafe_yyjson_is_str_noesc(str, len); + yyjson_subtype sub = noesc ? YYJSON_SUBTYPE_NOESC : YYJSON_SUBTYPE_NONE; + yyjson_mut_val *val = unsafe_yyjson_mut_val(doc, 1); + char *new_str = unsafe_yyjson_mut_strncpy(doc, str, len); + if (yyjson_likely(val && new_str)) { + val->tag = ((uint64_t)len << YYJSON_TAG_BIT) | + (uint64_t)(YYJSON_TYPE_STR | sub); + val->uni.str = new_str; + return val; + } + } return NULL; } @@ -4290,14 +5831,6 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_arr_get_last( * Mutable JSON Array Iterator API (Implementation) *============================================================================*/ -struct yyjson_mut_arr_iter { - size_t idx; /**< current index, from 0 */ - size_t max; /**< maximum index, idx < max */ - yyjson_mut_val *cur; /**< current value */ - yyjson_mut_val *pre; /**< previous value */ - yyjson_mut_val *arr; /**< the array being iterated */ -}; - yyjson_api_inline bool yyjson_mut_arr_iter_init(yyjson_mut_val *arr, yyjson_mut_arr_iter *iter) { if (yyjson_likely(yyjson_mut_is_arr(arr) && iter)) { @@ -4312,6 +5845,13 @@ yyjson_api_inline bool yyjson_mut_arr_iter_init(yyjson_mut_val *arr, return false; } +yyjson_api_inline yyjson_mut_arr_iter yyjson_mut_arr_iter_with( + yyjson_mut_val *arr) { + yyjson_mut_arr_iter iter; + yyjson_mut_arr_iter_init(arr, &iter); + return iter; +} + yyjson_api_inline bool yyjson_mut_arr_iter_has_next(yyjson_mut_arr_iter *iter) { return iter ? iter->idx < iter->max : false; } @@ -4386,7 +5926,8 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_arr(yyjson_mut_doc *doc) { yyjson_api_inline yyjson_mut_val *yyjson_mut_arr_with_bool( yyjson_mut_doc *doc, const bool *vals, size_t count) { yyjson_mut_arr_with_func({ - val->tag = YYJSON_TYPE_BOOL | (uint8_t)((uint8_t)vals[i] << 3); + bool _val = !!vals[i]; + val->tag = YYJSON_TYPE_BOOL | (uint8_t)((uint8_t)_val << 3); }); } @@ -4909,15 +6450,11 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_get(yyjson_mut_val *obj, yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_getn(yyjson_mut_val *obj, const char *_key, size_t key_len) { - uint64_t tag = (((uint64_t)key_len) << YYJSON_TAG_BIT) | YYJSON_TYPE_STR; size_t len = yyjson_mut_obj_size(obj); if (yyjson_likely(len && _key)) { yyjson_mut_val *key = ((yyjson_mut_val *)obj->uni.ptr)->next->next; while (len-- > 0) { - if (key->tag == tag && - memcmp(key->uni.ptr, _key, key_len) == 0) { - return key->next; - } + if (unsafe_yyjson_equals_strn(key, _key, key_len)) return key->next; key = key->next->next; } } @@ -4930,14 +6467,6 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_getn(yyjson_mut_val *obj, * Mutable JSON Object Iterator API (Implementation) *============================================================================*/ -struct yyjson_mut_obj_iter { - size_t idx; /**< current key index, from 0 */ - size_t max; /**< maximum key index, idx < max */ - yyjson_mut_val *cur; /**< current key */ - yyjson_mut_val *pre; /**< previous key */ - yyjson_mut_val *obj; /**< the object being iterated */ -}; - yyjson_api_inline bool yyjson_mut_obj_iter_init(yyjson_mut_val *obj, yyjson_mut_obj_iter *iter) { if (yyjson_likely(yyjson_mut_is_obj(obj) && iter)) { @@ -4952,6 +6481,13 @@ yyjson_api_inline bool yyjson_mut_obj_iter_init(yyjson_mut_val *obj, return false; } +yyjson_api_inline yyjson_mut_obj_iter yyjson_mut_obj_iter_with( + yyjson_mut_val *obj) { + yyjson_mut_obj_iter iter; + yyjson_mut_obj_iter_init(obj, &iter); + return iter; +} + yyjson_api_inline bool yyjson_mut_obj_iter_has_next(yyjson_mut_obj_iter *iter) { return iter ? iter->idx < iter->max : false; } @@ -4984,8 +6520,8 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_iter_remove( iter->max--; unsafe_yyjson_set_len(iter->obj, iter->max); prev->next->next = next; - iter->cur = next; - return cur; + iter->cur = prev; + return cur->next; } return NULL; } @@ -5004,8 +6540,7 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_iter_getn( while (idx++ < max) { pre = cur; cur = cur->next->next; - if (unsafe_yyjson_get_len(cur) == key_len && - memcmp(cur->uni.str, key, key_len) == 0) { + if (unsafe_yyjson_equals_strn(cur, key, key_len)) { iter->idx += idx; if (iter->idx > max) iter->idx -= max + 1; iter->pre = pre; @@ -5121,7 +6656,7 @@ yyjson_api_inline void unsafe_yyjson_mut_obj_add(yyjson_mut_val *obj, } yyjson_api_inline yyjson_mut_val *unsafe_yyjson_mut_obj_remove( - yyjson_mut_val *obj, const char *key, size_t key_len, uint64_t key_tag) { + yyjson_mut_val *obj, const char *key, size_t key_len) { size_t obj_len = unsafe_yyjson_get_len(obj); if (obj_len) { yyjson_mut_val *pre_key = (yyjson_mut_val *)obj->uni.ptr; @@ -5129,8 +6664,7 @@ yyjson_api_inline yyjson_mut_val *unsafe_yyjson_mut_obj_remove( yyjson_mut_val *removed_item = NULL; size_t i; for (i = 0; i < obj_len; i++) { - if (key_tag == cur_key->tag && - memcmp(key, cur_key->uni.ptr, key_len) == 0) { + if (unsafe_yyjson_equals_strn(cur_key, key, key_len)) { if (!removed_item) removed_item = cur_key->next; cur_key = cur_key->next->next; pre_key->next->next = cur_key; @@ -5159,8 +6693,7 @@ yyjson_api_inline bool unsafe_yyjson_mut_obj_replace(yyjson_mut_val *obj, yyjson_mut_val *cur_key = pre_key->next->next; size_t i; for (i = 0; i < obj_len; i++) { - if (key->tag == cur_key->tag && - memcmp(key->uni.str, cur_key->uni.ptr, key_len) == 0) { + if (unsafe_yyjson_equals_strn(cur_key, key->uni.str, key_len)) { cur_key->next->tag = val->tag; cur_key->next->uni.u64 = val->uni.u64; return true; @@ -5193,17 +6726,27 @@ yyjson_api_inline bool yyjson_mut_obj_add(yyjson_mut_val *obj, yyjson_api_inline bool yyjson_mut_obj_put(yyjson_mut_val *obj, yyjson_mut_val *key, yyjson_mut_val *val) { - if (yyjson_likely(yyjson_mut_is_obj(obj) && - yyjson_mut_is_str(key))) { - unsafe_yyjson_mut_obj_remove(obj, key->uni.str, - unsafe_yyjson_get_len(key), key->tag); - if (yyjson_likely(val)) { - unsafe_yyjson_mut_obj_add(obj, key, val, - unsafe_yyjson_get_len(obj)); + bool replaced = false; + size_t key_len; + yyjson_mut_obj_iter iter; + yyjson_mut_val *cur_key; + if (yyjson_unlikely(!yyjson_mut_is_obj(obj) || + !yyjson_mut_is_str(key))) return false; + key_len = unsafe_yyjson_get_len(key); + yyjson_mut_obj_iter_init(obj, &iter); + while ((cur_key = yyjson_mut_obj_iter_next(&iter)) != 0) { + if (unsafe_yyjson_equals_strn(cur_key, key->uni.str, key_len)) { + if (!replaced && val) { + replaced = true; + val->next = cur_key->next->next; + cur_key->next = val; + } else { + yyjson_mut_obj_iter_remove(&iter); + } } - return true; } - return false; + if (!replaced && val) unsafe_yyjson_mut_obj_add(obj, key, val, iter.max); + return true; } yyjson_api_inline bool yyjson_mut_obj_insert(yyjson_mut_val *obj, @@ -5232,8 +6775,7 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_remove(yyjson_mut_val *obj, yyjson_mut_val *key) { if (yyjson_likely(yyjson_mut_is_obj(obj) && yyjson_mut_is_str(key))) { return unsafe_yyjson_mut_obj_remove(obj, key->uni.str, - unsafe_yyjson_get_len(key), - key->tag); + unsafe_yyjson_get_len(key)); } return NULL; } @@ -5242,8 +6784,7 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_remove_key( yyjson_mut_val *obj, const char *key) { if (yyjson_likely(yyjson_mut_is_obj(obj) && key)) { size_t key_len = strlen(key); - uint64_t tag = ((uint64_t)key_len << YYJSON_TAG_BIT) | YYJSON_TYPE_STR; - return unsafe_yyjson_mut_obj_remove(obj, key, key_len, tag); + return unsafe_yyjson_mut_obj_remove(obj, key, key_len); } return NULL; } @@ -5251,8 +6792,7 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_remove_key( yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_remove_keyn( yyjson_mut_val *obj, const char *key, size_t key_len) { if (yyjson_likely(yyjson_mut_is_obj(obj) && key)) { - uint64_t tag = ((uint64_t)key_len << YYJSON_TAG_BIT) | YYJSON_TYPE_STR; - return unsafe_yyjson_mut_obj_remove(obj, key, key_len, tag); + return unsafe_yyjson_mut_obj_remove(obj, key, key_len); } return NULL; } @@ -5297,7 +6837,10 @@ yyjson_api_inline bool yyjson_mut_obj_rotate(yyjson_mut_val *obj, if (yyjson_likely(key)) { \ size_t len = unsafe_yyjson_get_len(obj); \ yyjson_mut_val *val = key + 1; \ - key->tag = YYJSON_TYPE_STR | YYJSON_SUBTYPE_NONE; \ + size_t key_len = strlen(_key); \ + bool noesc = unsafe_yyjson_is_str_noesc(_key, key_len); \ + key->tag = YYJSON_TYPE_STR; \ + key->tag |= noesc ? YYJSON_SUBTYPE_NOESC : YYJSON_SUBTYPE_NONE; \ key->tag |= (uint64_t)strlen(_key) << YYJSON_TAG_BIT; \ key->uni.str = _key; \ func \ @@ -5336,6 +6879,7 @@ yyjson_api_inline bool yyjson_mut_obj_add_bool(yyjson_mut_doc *doc, const char *_key, bool _val) { yyjson_mut_obj_add_func({ + _val = !!_val; val->tag = YYJSON_TYPE_BOOL | (uint8_t)((uint8_t)(_val) << 3); }); } @@ -5386,7 +6930,10 @@ yyjson_api_inline bool yyjson_mut_obj_add_str(yyjson_mut_doc *doc, const char *_val) { if (yyjson_unlikely(!_val)) return false; yyjson_mut_obj_add_func({ + size_t val_len = strlen(_val); + bool val_noesc = unsafe_yyjson_is_str_noesc(_val, val_len); val->tag = ((uint64_t)strlen(_val) << YYJSON_TAG_BIT) | YYJSON_TYPE_STR; + val->tag |= val_noesc ? YYJSON_SUBTYPE_NOESC : YYJSON_SUBTYPE_NONE; val->uni.str = _val; }); } @@ -5429,6 +6976,22 @@ yyjson_api_inline bool yyjson_mut_obj_add_strncpy(yyjson_mut_doc *doc, }); } +yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_add_arr(yyjson_mut_doc *doc, + yyjson_mut_val *obj, + const char *_key) { + yyjson_mut_val *key = yyjson_mut_str(doc, _key); + yyjson_mut_val *val = yyjson_mut_arr(doc); + return yyjson_mut_obj_add(obj, key, val) ? val : NULL; +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_add_obj(yyjson_mut_doc *doc, + yyjson_mut_val *obj, + const char *_key) { + yyjson_mut_val *key = yyjson_mut_str(doc, _key); + yyjson_mut_val *val = yyjson_mut_obj(doc); + return yyjson_mut_obj_add(obj, key, val) ? val : NULL; +} + yyjson_api_inline bool yyjson_mut_obj_add_val(yyjson_mut_doc *doc, yyjson_mut_val *obj, const char *_key, @@ -5452,8 +7015,7 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_remove_strn( yyjson_mut_val *val_removed = NULL; yyjson_mut_obj_iter_init(obj, &iter); while ((key = yyjson_mut_obj_iter_next(&iter)) != NULL) { - if (unsafe_yyjson_get_len(key) == _len && - memcmp(key->uni.str, _key, _len) == 0) { + if (unsafe_yyjson_equals_strn(key, _key, _len)) { if (!val_removed) val_removed = key->next; yyjson_mut_obj_iter_remove(&iter); } @@ -5463,71 +7025,880 @@ yyjson_api_inline yyjson_mut_val *yyjson_mut_obj_remove_strn( return NULL; } +yyjson_api_inline bool yyjson_mut_obj_rename_key(yyjson_mut_doc *doc, + yyjson_mut_val *obj, + const char *key, + const char *new_key) { + if (!key || !new_key) return false; + return yyjson_mut_obj_rename_keyn(doc, obj, key, strlen(key), + new_key, strlen(new_key)); +} + +yyjson_api_inline bool yyjson_mut_obj_rename_keyn(yyjson_mut_doc *doc, + yyjson_mut_val *obj, + const char *key, + size_t len, + const char *new_key, + size_t new_len) { + char *cpy_key = NULL; + yyjson_mut_val *old_key; + yyjson_mut_obj_iter iter; + if (!doc || !obj || !key || !new_key) return false; + yyjson_mut_obj_iter_init(obj, &iter); + while ((old_key = yyjson_mut_obj_iter_next(&iter))) { + if (unsafe_yyjson_equals_strn((void *)old_key, key, len)) { + if (!cpy_key) { + cpy_key = unsafe_yyjson_mut_strncpy(doc, new_key, new_len); + if (!cpy_key) return false; + } + yyjson_mut_set_strn(old_key, cpy_key, new_len); + } + } + return cpy_key != NULL; +} + /*============================================================================== * JSON Pointer API (Implementation) *============================================================================*/ -/* `val` not null, `ptr` start with '/', `len` > 0. */ -yyjson_api yyjson_val *unsafe_yyjson_get_pointer(yyjson_val *val, - const char *ptr, - size_t len); +#define yyjson_ptr_set_err(_code, _msg) do { \ + if (err) { \ + err->code = YYJSON_PTR_ERR_##_code; \ + err->msg = _msg; \ + err->pos = 0; \ + } \ +} while(false) -/* `val` not null, `ptr` start with '/', `len` > 0. */ -yyjson_api yyjson_mut_val *unsafe_yyjson_mut_get_pointer(yyjson_mut_val *val, +/* require: val != NULL, *ptr == '/', len > 0 */ +yyjson_api yyjson_val *unsafe_yyjson_ptr_getx(yyjson_val *val, + const char *ptr, size_t len, + yyjson_ptr_err *err); + +/* require: val != NULL, *ptr == '/', len > 0 */ +yyjson_api yyjson_mut_val *unsafe_yyjson_mut_ptr_getx(yyjson_mut_val *val, + const char *ptr, + size_t len, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err); + +/* require: val/new_val/doc != NULL, *ptr == '/', len > 0 */ +yyjson_api bool unsafe_yyjson_mut_ptr_putx(yyjson_mut_val *val, + const char *ptr, size_t len, + yyjson_mut_val *new_val, + yyjson_mut_doc *doc, + bool create_parent, bool insert_new, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err); + +/* require: val/err != NULL, *ptr == '/', len > 0 */ +yyjson_api yyjson_mut_val *unsafe_yyjson_mut_ptr_replacex( + yyjson_mut_val *val, const char *ptr, size_t len, yyjson_mut_val *new_val, + yyjson_ptr_ctx *ctx, yyjson_ptr_err *err); + +/* require: val/err != NULL, *ptr == '/', len > 0 */ +yyjson_api yyjson_mut_val *unsafe_yyjson_mut_ptr_removex(yyjson_mut_val *val, const char *ptr, - size_t len); + size_t len, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err); -yyjson_api_inline yyjson_val *yyjson_get_pointern(yyjson_val *val, - const char *ptr, - size_t len) { - if (!val || !ptr) return NULL; - if (len == 0) return val; - if (*ptr != '/') return NULL; - return unsafe_yyjson_get_pointer(val, ptr, len); +yyjson_api_inline yyjson_val *yyjson_doc_ptr_get(yyjson_doc *doc, + const char *ptr) { + if (yyjson_unlikely(!ptr)) return NULL; + return yyjson_doc_ptr_getn(doc, ptr, strlen(ptr)); } -yyjson_api_inline yyjson_val *yyjson_get_pointer(yyjson_val *val, - const char *ptr) { - if (!val || !ptr) return NULL; - return yyjson_get_pointern(val, ptr, strlen(ptr)); +yyjson_api_inline yyjson_val *yyjson_doc_ptr_getn(yyjson_doc *doc, + const char *ptr, size_t len) { + return yyjson_doc_ptr_getx(doc, ptr, len, NULL); } -yyjson_api_inline yyjson_val *yyjson_doc_get_pointern(yyjson_doc *doc, +yyjson_api_inline yyjson_val *yyjson_doc_ptr_getx(yyjson_doc *doc, + const char *ptr, size_t len, + yyjson_ptr_err *err) { + yyjson_ptr_set_err(NONE, NULL); + if (yyjson_unlikely(!doc || !ptr)) { + yyjson_ptr_set_err(PARAMETER, "input parameter is NULL"); + return NULL; + } + if (yyjson_unlikely(!doc->root)) { + yyjson_ptr_set_err(NULL_ROOT, "document's root is NULL"); + return NULL; + } + if (yyjson_unlikely(len == 0)) { + return doc->root; + } + if (yyjson_unlikely(*ptr != '/')) { + yyjson_ptr_set_err(SYNTAX, "no prefix '/'"); + return NULL; + } + return unsafe_yyjson_ptr_getx(doc->root, ptr, len, err); +} + +yyjson_api_inline yyjson_val *yyjson_ptr_get(yyjson_val *val, + const char *ptr) { + if (yyjson_unlikely(!ptr)) return NULL; + return yyjson_ptr_getn(val, ptr, strlen(ptr)); +} + +yyjson_api_inline yyjson_val *yyjson_ptr_getn(yyjson_val *val, + const char *ptr, size_t len) { + return yyjson_ptr_getx(val, ptr, len, NULL); +} + +yyjson_api_inline yyjson_val *yyjson_ptr_getx(yyjson_val *val, + const char *ptr, size_t len, + yyjson_ptr_err *err) { + yyjson_ptr_set_err(NONE, NULL); + if (yyjson_unlikely(!val || !ptr)) { + yyjson_ptr_set_err(PARAMETER, "input parameter is NULL"); + return NULL; + } + if (yyjson_unlikely(len == 0)) { + return val; + } + if (yyjson_unlikely(*ptr != '/')) { + yyjson_ptr_set_err(SYNTAX, "no prefix '/'"); + return NULL; + } + return unsafe_yyjson_ptr_getx(val, ptr, len, err); +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_ptr_get(yyjson_mut_doc *doc, + const char *ptr) { + if (!ptr) return NULL; + return yyjson_mut_doc_ptr_getn(doc, ptr, strlen(ptr)); +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_ptr_getn(yyjson_mut_doc *doc, + const char *ptr, + size_t len) { + return yyjson_mut_doc_ptr_getx(doc, ptr, len, NULL, NULL); +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_ptr_getx(yyjson_mut_doc *doc, + const char *ptr, + size_t len, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err) { + yyjson_ptr_set_err(NONE, NULL); + if (ctx) memset(ctx, 0, sizeof(*ctx)); + + if (yyjson_unlikely(!doc || !ptr)) { + yyjson_ptr_set_err(PARAMETER, "input parameter is NULL"); + return NULL; + } + if (yyjson_unlikely(!doc->root)) { + yyjson_ptr_set_err(NULL_ROOT, "document's root is NULL"); + return NULL; + } + if (yyjson_unlikely(len == 0)) { + return doc->root; + } + if (yyjson_unlikely(*ptr != '/')) { + yyjson_ptr_set_err(SYNTAX, "no prefix '/'"); + return NULL; + } + return unsafe_yyjson_mut_ptr_getx(doc->root, ptr, len, ctx, err); +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_ptr_get(yyjson_mut_val *val, + const char *ptr) { + if (!ptr) return NULL; + return yyjson_mut_ptr_getn(val, ptr, strlen(ptr)); +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_ptr_getn(yyjson_mut_val *val, const char *ptr, size_t len) { - return yyjson_get_pointern(doc ? doc->root : NULL, ptr, len); + return yyjson_mut_ptr_getx(val, ptr, len, NULL, NULL); +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_ptr_getx(yyjson_mut_val *val, + const char *ptr, + size_t len, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err) { + yyjson_ptr_set_err(NONE, NULL); + if (ctx) memset(ctx, 0, sizeof(*ctx)); + + if (yyjson_unlikely(!val || !ptr)) { + yyjson_ptr_set_err(PARAMETER, "input parameter is NULL"); + return NULL; + } + if (yyjson_unlikely(len == 0)) { + return val; + } + if (yyjson_unlikely(*ptr != '/')) { + yyjson_ptr_set_err(SYNTAX, "no prefix '/'"); + return NULL; + } + return unsafe_yyjson_mut_ptr_getx(val, ptr, len, ctx, err); } +yyjson_api_inline bool yyjson_mut_doc_ptr_add(yyjson_mut_doc *doc, + const char *ptr, + yyjson_mut_val *new_val) { + if (yyjson_unlikely(!ptr)) return false; + return yyjson_mut_doc_ptr_addn(doc, ptr, strlen(ptr), new_val); +} + +yyjson_api_inline bool yyjson_mut_doc_ptr_addn(yyjson_mut_doc *doc, + const char *ptr, + size_t len, + yyjson_mut_val *new_val) { + return yyjson_mut_doc_ptr_addx(doc, ptr, len, new_val, true, NULL, NULL); +} + +yyjson_api_inline bool yyjson_mut_doc_ptr_addx(yyjson_mut_doc *doc, + const char *ptr, size_t len, + yyjson_mut_val *new_val, + bool create_parent, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err) { + yyjson_ptr_set_err(NONE, NULL); + if (ctx) memset(ctx, 0, sizeof(*ctx)); + + if (yyjson_unlikely(!doc || !ptr || !new_val)) { + yyjson_ptr_set_err(PARAMETER, "input parameter is NULL"); + return false; + } + if (yyjson_unlikely(len == 0)) { + if (doc->root) { + yyjson_ptr_set_err(SET_ROOT, "cannot set document's root"); + return false; + } else { + doc->root = new_val; + return true; + } + } + if (yyjson_unlikely(*ptr != '/')) { + yyjson_ptr_set_err(SYNTAX, "no prefix '/'"); + return false; + } + if (yyjson_unlikely(!doc->root && !create_parent)) { + yyjson_ptr_set_err(NULL_ROOT, "document's root is NULL"); + return false; + } + if (yyjson_unlikely(!doc->root)) { + yyjson_mut_val *root = yyjson_mut_obj(doc); + if (yyjson_unlikely(!root)) { + yyjson_ptr_set_err(MEMORY_ALLOCATION, "failed to create value"); + return false; + } + if (unsafe_yyjson_mut_ptr_putx(root, ptr, len, new_val, doc, + create_parent, true, ctx, err)) { + doc->root = root; + return true; + } + return false; + } + return unsafe_yyjson_mut_ptr_putx(doc->root, ptr, len, new_val, doc, + create_parent, true, ctx, err); +} + +yyjson_api_inline bool yyjson_mut_ptr_add(yyjson_mut_val *val, + const char *ptr, + yyjson_mut_val *new_val, + yyjson_mut_doc *doc) { + if (yyjson_unlikely(!ptr)) return false; + return yyjson_mut_ptr_addn(val, ptr, strlen(ptr), new_val, doc); +} + +yyjson_api_inline bool yyjson_mut_ptr_addn(yyjson_mut_val *val, + const char *ptr, size_t len, + yyjson_mut_val *new_val, + yyjson_mut_doc *doc) { + return yyjson_mut_ptr_addx(val, ptr, len, new_val, doc, true, NULL, NULL); +} + +yyjson_api_inline bool yyjson_mut_ptr_addx(yyjson_mut_val *val, + const char *ptr, size_t len, + yyjson_mut_val *new_val, + yyjson_mut_doc *doc, + bool create_parent, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err) { + yyjson_ptr_set_err(NONE, NULL); + if (ctx) memset(ctx, 0, sizeof(*ctx)); + + if (yyjson_unlikely(!val || !ptr || !new_val || !doc)) { + yyjson_ptr_set_err(PARAMETER, "input parameter is NULL"); + return false; + } + if (yyjson_unlikely(len == 0)) { + yyjson_ptr_set_err(SET_ROOT, "cannot set root"); + return false; + } + if (yyjson_unlikely(*ptr != '/')) { + yyjson_ptr_set_err(SYNTAX, "no prefix '/'"); + return false; + } + return unsafe_yyjson_mut_ptr_putx(val, ptr, len, new_val, + doc, create_parent, true, ctx, err); +} + +yyjson_api_inline bool yyjson_mut_doc_ptr_set(yyjson_mut_doc *doc, + const char *ptr, + yyjson_mut_val *new_val) { + if (yyjson_unlikely(!ptr)) return false; + return yyjson_mut_doc_ptr_setn(doc, ptr, strlen(ptr), new_val); +} + +yyjson_api_inline bool yyjson_mut_doc_ptr_setn(yyjson_mut_doc *doc, + const char *ptr, size_t len, + yyjson_mut_val *new_val) { + return yyjson_mut_doc_ptr_setx(doc, ptr, len, new_val, true, NULL, NULL); +} + +yyjson_api_inline bool yyjson_mut_doc_ptr_setx(yyjson_mut_doc *doc, + const char *ptr, size_t len, + yyjson_mut_val *new_val, + bool create_parent, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err) { + yyjson_ptr_set_err(NONE, NULL); + if (ctx) memset(ctx, 0, sizeof(*ctx)); + + if (yyjson_unlikely(!doc || !ptr)) { + yyjson_ptr_set_err(PARAMETER, "input parameter is NULL"); + return false; + } + if (yyjson_unlikely(len == 0)) { + if (ctx) ctx->old = doc->root; + doc->root = new_val; + return true; + } + if (yyjson_unlikely(*ptr != '/')) { + yyjson_ptr_set_err(SYNTAX, "no prefix '/'"); + return false; + } + if (!new_val) { + if (!doc->root) { + yyjson_ptr_set_err(RESOLVE, "JSON pointer cannot be resolved"); + return false; + } + return !!unsafe_yyjson_mut_ptr_removex(doc->root, ptr, len, ctx, err); + } + if (yyjson_unlikely(!doc->root && !create_parent)) { + yyjson_ptr_set_err(NULL_ROOT, "document's root is NULL"); + return false; + } + if (yyjson_unlikely(!doc->root)) { + yyjson_mut_val *root = yyjson_mut_obj(doc); + if (yyjson_unlikely(!root)) { + yyjson_ptr_set_err(MEMORY_ALLOCATION, "failed to create value"); + return false; + } + if (unsafe_yyjson_mut_ptr_putx(root, ptr, len, new_val, doc, + create_parent, false, ctx, err)) { + doc->root = root; + return true; + } + return false; + } + return unsafe_yyjson_mut_ptr_putx(doc->root, ptr, len, new_val, doc, + create_parent, false, ctx, err); +} + +yyjson_api_inline bool yyjson_mut_ptr_set(yyjson_mut_val *val, + const char *ptr, + yyjson_mut_val *new_val, + yyjson_mut_doc *doc) { + if (yyjson_unlikely(!ptr)) return false; + return yyjson_mut_ptr_setn(val, ptr, strlen(ptr), new_val, doc); +} + +yyjson_api_inline bool yyjson_mut_ptr_setn(yyjson_mut_val *val, + const char *ptr, size_t len, + yyjson_mut_val *new_val, + yyjson_mut_doc *doc) { + return yyjson_mut_ptr_setx(val, ptr, len, new_val, doc, true, NULL, NULL); +} + +yyjson_api_inline bool yyjson_mut_ptr_setx(yyjson_mut_val *val, + const char *ptr, size_t len, + yyjson_mut_val *new_val, + yyjson_mut_doc *doc, + bool create_parent, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err) { + yyjson_ptr_set_err(NONE, NULL); + if (ctx) memset(ctx, 0, sizeof(*ctx)); + + if (yyjson_unlikely(!val || !ptr || !doc)) { + yyjson_ptr_set_err(PARAMETER, "input parameter is NULL"); + return false; + } + if (yyjson_unlikely(len == 0)) { + yyjson_ptr_set_err(SET_ROOT, "cannot set root"); + return false; + } + if (yyjson_unlikely(*ptr != '/')) { + yyjson_ptr_set_err(SYNTAX, "no prefix '/'"); + return false; + } + if (!new_val) { + return !!unsafe_yyjson_mut_ptr_removex(val, ptr, len, ctx, err); + } + return unsafe_yyjson_mut_ptr_putx(val, ptr, len, new_val, doc, + create_parent, false, ctx, err); +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_ptr_replace( + yyjson_mut_doc *doc, const char *ptr, yyjson_mut_val *new_val) { + if (!ptr) return NULL; + return yyjson_mut_doc_ptr_replacen(doc, ptr, strlen(ptr), new_val); +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_ptr_replacen( + yyjson_mut_doc *doc, const char *ptr, size_t len, yyjson_mut_val *new_val) { + return yyjson_mut_doc_ptr_replacex(doc, ptr, len, new_val, NULL, NULL); +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_ptr_replacex( + yyjson_mut_doc *doc, const char *ptr, size_t len, yyjson_mut_val *new_val, + yyjson_ptr_ctx *ctx, yyjson_ptr_err *err) { + + yyjson_ptr_set_err(NONE, NULL); + if (ctx) memset(ctx, 0, sizeof(*ctx)); + + if (yyjson_unlikely(!doc || !ptr || !new_val)) { + yyjson_ptr_set_err(PARAMETER, "input parameter is NULL"); + return NULL; + } + if (yyjson_unlikely(len == 0)) { + yyjson_mut_val *root = doc->root; + if (yyjson_unlikely(!root)) { + yyjson_ptr_set_err(RESOLVE, "JSON pointer cannot be resolved"); + return NULL; + } + if (ctx) ctx->old = root; + doc->root = new_val; + return root; + } + if (yyjson_unlikely(!doc->root)) { + yyjson_ptr_set_err(NULL_ROOT, "document's root is NULL"); + return NULL; + } + if (yyjson_unlikely(*ptr != '/')) { + yyjson_ptr_set_err(SYNTAX, "no prefix '/'"); + return NULL; + } + return unsafe_yyjson_mut_ptr_replacex(doc->root, ptr, len, new_val, + ctx, err); +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_ptr_replace( + yyjson_mut_val *val, const char *ptr, yyjson_mut_val *new_val) { + if (!ptr) return NULL; + return yyjson_mut_ptr_replacen(val, ptr, strlen(ptr), new_val); +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_ptr_replacen( + yyjson_mut_val *val, const char *ptr, size_t len, yyjson_mut_val *new_val) { + return yyjson_mut_ptr_replacex(val, ptr, len, new_val, NULL, NULL); +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_ptr_replacex( + yyjson_mut_val *val, const char *ptr, size_t len, yyjson_mut_val *new_val, + yyjson_ptr_ctx *ctx, yyjson_ptr_err *err) { + + yyjson_ptr_set_err(NONE, NULL); + if (ctx) memset(ctx, 0, sizeof(*ctx)); + + if (yyjson_unlikely(!val || !ptr || !new_val)) { + yyjson_ptr_set_err(PARAMETER, "input parameter is NULL"); + return NULL; + } + if (yyjson_unlikely(len == 0)) { + yyjson_ptr_set_err(SET_ROOT, "cannot set root"); + return NULL; + } + if (yyjson_unlikely(*ptr != '/')) { + yyjson_ptr_set_err(SYNTAX, "no prefix '/'"); + return NULL; + } + return unsafe_yyjson_mut_ptr_replacex(val, ptr, len, new_val, ctx, err); +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_ptr_remove( + yyjson_mut_doc *doc, const char *ptr) { + if (!ptr) return NULL; + return yyjson_mut_doc_ptr_removen(doc, ptr, strlen(ptr)); +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_ptr_removen( + yyjson_mut_doc *doc, const char *ptr, size_t len) { + return yyjson_mut_doc_ptr_removex(doc, ptr, len, NULL, NULL); +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_ptr_removex( + yyjson_mut_doc *doc, const char *ptr, size_t len, + yyjson_ptr_ctx *ctx, yyjson_ptr_err *err) { + + yyjson_ptr_set_err(NONE, NULL); + if (ctx) memset(ctx, 0, sizeof(*ctx)); + + if (yyjson_unlikely(!doc || !ptr)) { + yyjson_ptr_set_err(PARAMETER, "input parameter is NULL"); + return NULL; + } + if (yyjson_unlikely(!doc->root)) { + yyjson_ptr_set_err(NULL_ROOT, "document's root is NULL"); + return NULL; + } + if (yyjson_unlikely(len == 0)) { + yyjson_mut_val *root = doc->root; + if (ctx) ctx->old = root; + doc->root = NULL; + return root; + } + if (yyjson_unlikely(*ptr != '/')) { + yyjson_ptr_set_err(SYNTAX, "no prefix '/'"); + return NULL; + } + return unsafe_yyjson_mut_ptr_removex(doc->root, ptr, len, ctx, err); +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_ptr_remove(yyjson_mut_val *val, + const char *ptr) { + if (!ptr) return NULL; + return yyjson_mut_ptr_removen(val, ptr, strlen(ptr)); +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_ptr_removen(yyjson_mut_val *val, + const char *ptr, + size_t len) { + return yyjson_mut_ptr_removex(val, ptr, len, NULL, NULL); +} + +yyjson_api_inline yyjson_mut_val *yyjson_mut_ptr_removex(yyjson_mut_val *val, + const char *ptr, + size_t len, + yyjson_ptr_ctx *ctx, + yyjson_ptr_err *err) { + yyjson_ptr_set_err(NONE, NULL); + if (ctx) memset(ctx, 0, sizeof(*ctx)); + + if (yyjson_unlikely(!val || !ptr)) { + yyjson_ptr_set_err(PARAMETER, "input parameter is NULL"); + return NULL; + } + if (yyjson_unlikely(len == 0)) { + yyjson_ptr_set_err(SET_ROOT, "cannot set root"); + return NULL; + } + if (yyjson_unlikely(*ptr != '/')) { + yyjson_ptr_set_err(SYNTAX, "no prefix '/'"); + return NULL; + } + return unsafe_yyjson_mut_ptr_removex(val, ptr, len, ctx, err); +} + +yyjson_api_inline bool yyjson_ptr_ctx_append(yyjson_ptr_ctx *ctx, + yyjson_mut_val *key, + yyjson_mut_val *val) { + yyjson_mut_val *ctn, *pre_key, *pre_val, *cur_key, *cur_val; + if (!ctx || !ctx->ctn || !val) return false; + ctn = ctx->ctn; + + if (yyjson_mut_is_obj(ctn)) { + if (!key) return false; + key->next = val; + pre_key = ctx->pre; + if (unsafe_yyjson_get_len(ctn) == 0) { + val->next = key; + ctn->uni.ptr = key; + ctx->pre = key; + } else if (!pre_key) { + pre_key = (yyjson_mut_val *)ctn->uni.ptr; + pre_val = pre_key->next; + val->next = pre_val->next; + pre_val->next = key; + ctn->uni.ptr = key; + ctx->pre = pre_key; + } else { + cur_key = pre_key->next->next; + cur_val = cur_key->next; + val->next = cur_val->next; + cur_val->next = key; + if (ctn->uni.ptr == cur_key) ctn->uni.ptr = key; + ctx->pre = cur_key; + } + } else { + pre_val = ctx->pre; + if (unsafe_yyjson_get_len(ctn) == 0) { + val->next = val; + ctn->uni.ptr = val; + ctx->pre = val; + } else if (!pre_val) { + pre_val = (yyjson_mut_val *)ctn->uni.ptr; + val->next = pre_val->next; + pre_val->next = val; + ctn->uni.ptr = val; + ctx->pre = pre_val; + } else { + cur_val = pre_val->next; + val->next = cur_val->next; + cur_val->next = val; + if (ctn->uni.ptr == cur_val) ctn->uni.ptr = val; + ctx->pre = cur_val; + } + } + unsafe_yyjson_inc_len(ctn); + return true; +} + +yyjson_api_inline bool yyjson_ptr_ctx_replace(yyjson_ptr_ctx *ctx, + yyjson_mut_val *val) { + yyjson_mut_val *ctn, *pre_key, *cur_key, *pre_val, *cur_val; + if (!ctx || !ctx->ctn || !ctx->pre || !val) return false; + ctn = ctx->ctn; + if (yyjson_mut_is_obj(ctn)) { + pre_key = ctx->pre; + pre_val = pre_key->next; + cur_key = pre_val->next; + cur_val = cur_key->next; + /* replace current value */ + cur_key->next = val; + val->next = cur_val->next; + ctx->old = cur_val; + } else { + pre_val = ctx->pre; + cur_val = pre_val->next; + /* replace current value */ + if (pre_val != cur_val) { + val->next = cur_val->next; + pre_val->next = val; + if (ctn->uni.ptr == cur_val) ctn->uni.ptr = val; + } else { + val->next = val; + ctn->uni.ptr = val; + ctx->pre = val; + } + ctx->old = cur_val; + } + return true; +} + +yyjson_api_inline bool yyjson_ptr_ctx_remove(yyjson_ptr_ctx *ctx) { + yyjson_mut_val *ctn, *pre_key, *pre_val, *cur_key, *cur_val; + size_t len; + if (!ctx || !ctx->ctn || !ctx->pre) return false; + ctn = ctx->ctn; + if (yyjson_mut_is_obj(ctn)) { + pre_key = ctx->pre; + pre_val = pre_key->next; + cur_key = pre_val->next; + cur_val = cur_key->next; + /* remove current key-value */ + pre_val->next = cur_val->next; + if (ctn->uni.ptr == cur_key) ctn->uni.ptr = pre_key; + ctx->pre = NULL; + ctx->old = cur_val; + } else { + pre_val = ctx->pre; + cur_val = pre_val->next; + /* remove current key-value */ + pre_val->next = cur_val->next; + if (ctn->uni.ptr == cur_val) ctn->uni.ptr = pre_val; + ctx->pre = NULL; + ctx->old = cur_val; + } + len = unsafe_yyjson_get_len(ctn) - 1; + if (len == 0) ctn->uni.ptr = NULL; + unsafe_yyjson_set_len(ctn, len); + return true; +} + +#undef yyjson_ptr_set_err + + + +/*============================================================================== + * JSON Value at Pointer API (Implementation) + *============================================================================*/ + +/** + Set provided `value` if the JSON Pointer (RFC 6901) exists and is type bool. + Returns true if value at `ptr` exists and is the correct type, otherwise false. + */ +yyjson_api_inline bool yyjson_ptr_get_bool( + yyjson_val *root, const char *ptr, bool *value) { + yyjson_val *val = yyjson_ptr_get(root, ptr); + if (value && yyjson_is_bool(val)) { + *value = unsafe_yyjson_get_bool(val); + return true; + } else { + return false; + } +} + +/** + Set provided `value` if the JSON Pointer (RFC 6901) exists and is an integer + that fits in `uint64_t`. Returns true if successful, otherwise false. + */ +yyjson_api_inline bool yyjson_ptr_get_uint( + yyjson_val *root, const char *ptr, uint64_t *value) { + yyjson_val *val = yyjson_ptr_get(root, ptr); + if (value && val) { + uint64_t ret = val->uni.u64; + if (unsafe_yyjson_is_uint(val) || + (unsafe_yyjson_is_sint(val) && !(ret >> 63))) { + *value = ret; + return true; + } + } + return false; +} + +/** + Set provided `value` if the JSON Pointer (RFC 6901) exists and is an integer + that fits in `int64_t`. Returns true if successful, otherwise false. + */ +yyjson_api_inline bool yyjson_ptr_get_sint( + yyjson_val *root, const char *ptr, int64_t *value) { + yyjson_val *val = yyjson_ptr_get(root, ptr); + if (value && val) { + int64_t ret = val->uni.i64; + if (unsafe_yyjson_is_sint(val) || + (unsafe_yyjson_is_uint(val) && ret >= 0)) { + *value = ret; + return true; + } + } + return false; +} + +/** + Set provided `value` if the JSON Pointer (RFC 6901) exists and is type real. + Returns true if value at `ptr` exists and is the correct type, otherwise false. + */ +yyjson_api_inline bool yyjson_ptr_get_real( + yyjson_val *root, const char *ptr, double *value) { + yyjson_val *val = yyjson_ptr_get(root, ptr); + if (value && yyjson_is_real(val)) { + *value = unsafe_yyjson_get_real(val); + return true; + } else { + return false; + } +} + +/** + Set provided `value` if the JSON Pointer (RFC 6901) exists and is type sint, + uint or real. + Returns true if value at `ptr` exists and is the correct type, otherwise false. + */ +yyjson_api_inline bool yyjson_ptr_get_num( + yyjson_val *root, const char *ptr, double *value) { + yyjson_val *val = yyjson_ptr_get(root, ptr); + if (value && yyjson_is_num(val)) { + *value = unsafe_yyjson_get_num(val); + return true; + } else { + return false; + } +} + +/** + Set provided `value` if the JSON Pointer (RFC 6901) exists and is type string. + Returns true if value at `ptr` exists and is the correct type, otherwise false. + */ +yyjson_api_inline bool yyjson_ptr_get_str( + yyjson_val *root, const char *ptr, const char **value) { + yyjson_val *val = yyjson_ptr_get(root, ptr); + if (value && yyjson_is_str(val)) { + *value = unsafe_yyjson_get_str(val); + return true; + } else { + return false; + } +} + + + +/*============================================================================== + * Deprecated + *============================================================================*/ + +/** @deprecated renamed to `yyjson_doc_ptr_get` */ +yyjson_deprecated("renamed to yyjson_doc_ptr_get") yyjson_api_inline yyjson_val *yyjson_doc_get_pointer(yyjson_doc *doc, const char *ptr) { - return yyjson_get_pointer(doc ? doc->root : NULL, ptr); + return yyjson_doc_ptr_get(doc, ptr); } -yyjson_api_inline yyjson_mut_val *yyjson_mut_get_pointern(yyjson_mut_val *val, - const char *ptr, - size_t len) { - if (!val || !ptr) return NULL; - if (len == 0) return val; - if (*ptr != '/') return NULL; - return unsafe_yyjson_mut_get_pointer(val, ptr, len); +/** @deprecated renamed to `yyjson_doc_ptr_getn` */ +yyjson_deprecated("renamed to yyjson_doc_ptr_getn") +yyjson_api_inline yyjson_val *yyjson_doc_get_pointern(yyjson_doc *doc, + const char *ptr, + size_t len) { + return yyjson_doc_ptr_getn(doc, ptr, len); } -yyjson_api_inline yyjson_mut_val *yyjson_mut_get_pointer(yyjson_mut_val *val, - const char *ptr) { - if (!val || !ptr) return NULL; - return yyjson_mut_get_pointern(val, ptr, strlen(ptr)); +/** @deprecated renamed to `yyjson_mut_doc_ptr_get` */ +yyjson_deprecated("renamed to yyjson_mut_doc_ptr_get") +yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_get_pointer( + yyjson_mut_doc *doc, const char *ptr) { + return yyjson_mut_doc_ptr_get(doc, ptr); } +/** @deprecated renamed to `yyjson_mut_doc_ptr_getn` */ +yyjson_deprecated("renamed to yyjson_mut_doc_ptr_getn") yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_get_pointern( yyjson_mut_doc *doc, const char *ptr, size_t len) { - return yyjson_mut_get_pointern(doc ? doc->root : NULL, ptr, len); + return yyjson_mut_doc_ptr_getn(doc, ptr, len); } -yyjson_api_inline yyjson_mut_val *yyjson_mut_doc_get_pointer( - yyjson_mut_doc *doc, const char *ptr) { - return yyjson_mut_get_pointer(doc ? doc->root : NULL, ptr); +/** @deprecated renamed to `yyjson_ptr_get` */ +yyjson_deprecated("renamed to yyjson_ptr_get") +yyjson_api_inline yyjson_val *yyjson_get_pointer(yyjson_val *val, + const char *ptr) { + return yyjson_ptr_get(val, ptr); +} + +/** @deprecated renamed to `yyjson_ptr_getn` */ +yyjson_deprecated("renamed to yyjson_ptr_getn") +yyjson_api_inline yyjson_val *yyjson_get_pointern(yyjson_val *val, + const char *ptr, + size_t len) { + return yyjson_ptr_getn(val, ptr, len); +} + +/** @deprecated renamed to `yyjson_mut_ptr_get` */ +yyjson_deprecated("renamed to yyjson_mut_ptr_get") +yyjson_api_inline yyjson_mut_val *yyjson_mut_get_pointer(yyjson_mut_val *val, + const char *ptr) { + return yyjson_mut_ptr_get(val, ptr); +} + +/** @deprecated renamed to `yyjson_mut_ptr_getn` */ +yyjson_deprecated("renamed to yyjson_mut_ptr_getn") +yyjson_api_inline yyjson_mut_val *yyjson_mut_get_pointern(yyjson_mut_val *val, + const char *ptr, + size_t len) { + return yyjson_mut_ptr_getn(val, ptr, len); +} + +/** @deprecated renamed to `yyjson_mut_ptr_getn` */ +yyjson_deprecated("renamed to unsafe_yyjson_ptr_getn") +yyjson_api_inline yyjson_val *unsafe_yyjson_get_pointer(yyjson_val *val, + const char *ptr, + size_t len) { + yyjson_ptr_err err; + return unsafe_yyjson_ptr_getx(val, ptr, len, &err); +} + +/** @deprecated renamed to `unsafe_yyjson_mut_ptr_getx` */ +yyjson_deprecated("renamed to unsafe_yyjson_mut_ptr_getx") +yyjson_api_inline yyjson_mut_val *unsafe_yyjson_mut_get_pointer( + yyjson_mut_val *val, const char *ptr, size_t len) { + yyjson_ptr_err err; + return unsafe_yyjson_mut_ptr_getx(val, ptr, len, NULL, &err); } diff --git a/integration/client b/integration/client index 429c03f5..77b72850 100755 --- a/integration/client +++ b/integration/client @@ -1,4 +1,6 @@ #!/usr/bin/env python3 +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2020-2023) import asyncio import sys @@ -9,10 +11,10 @@ import httpx port = sys.argv[1] url = f"http://127.0.0.1:{port}" -timeout = httpx.Timeout(5.0, connect_timeout=5.0) +timeout = httpx.Timeout(5.0) client = httpx.AsyncClient(timeout=timeout) -stop_time = time.time() + 15 +stop_time = time.time() + 5 TEST_MESSAGE = "http test running..." @@ -27,4 +29,7 @@ async def main(): sys.stdout.write(f"\r{TEST_MESSAGE} ok, {count} requests made\n") -asyncio.get_event_loop().run_until_complete(main()) +loop = asyncio.new_event_loop() +asyncio.set_event_loop(loop) +asyncio.run(main()) +loop.close() diff --git a/integration/init b/integration/init new file mode 100755 index 00000000..3b57fbb8 --- /dev/null +++ b/integration/init @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2023-2025) + +import multiprocessing.pool +import sys + +import orjson + +NUM_PROC = 16 + +TEST_MESSAGE = "parallel import of orjson running..." + + +class Custom: + pass + + +def default(_): + return None + + +def func(_): + orjson.dumps(Custom(), option=orjson.OPT_SERIALIZE_NUMPY, default=default) + orjson.loads(b'{"a":1,"b":2,"c":3}') + + +def main(): + sys.stdout.write(TEST_MESSAGE) + sys.stdout.flush() + with multiprocessing.pool.ThreadPool(processes=NUM_PROC) as pool: + pool.map(func, (i for i in range(NUM_PROC))) + sys.stdout.write(f"\r{TEST_MESSAGE} ok\n") + sys.stdout.flush() + + +if __name__ == "__main__": + main() diff --git a/integration/requirements.txt b/integration/requirements.txt index 7479be65..71e82086 100644 --- a/integration/requirements.txt +++ b/integration/requirements.txt @@ -1,3 +1,3 @@ flask;sys_platform!="win" gunicorn;sys_platform!="win" -httpx==0.14.3;sys_platform!="win" +httpx==0.28.1;sys_platform!="win" diff --git a/integration/run b/integration/run index 5094dd9a..af6453df 100755 --- a/integration/run +++ b/integration/run @@ -1,10 +1,14 @@ #!/usr/bin/env bash +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2018-2023), Eric Jolibois (2022) set -eou pipefail _dir="$(dirname "${BASH_SOURCE[0]}")" -to_run="${@:-thread http}" +to_run="${@:-thread http init}" + +export PYTHONMALLOC="debug" if [[ $to_run == *"thread"* ]]; then "${_dir}"/thread @@ -23,3 +27,7 @@ if [[ $to_run == *"typestubs"* ]]; then python "${_dir}"/typestubs.py mypy "${_dir}"/typestubs.py fi + +if [[ $to_run == *"init"* ]]; then + "${_dir}"/init +fi diff --git a/integration/thread b/integration/thread index b2e8d7f3..bc737787 100755 --- a/integration/thread +++ b/integration/thread @@ -1,15 +1,15 @@ #!/usr/bin/env python3 +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2018-2025) import sys import traceback - from concurrent.futures import ThreadPoolExecutor from operator import itemgetter from threading import get_ident import orjson - DATA = sorted( [ { @@ -39,13 +39,12 @@ def test_func(n): try: assert sorted(orjson.loads(orjson.dumps(DATA)), key=itemgetter("id")) == DATA except Exception: - STATUS = 1 traceback.print_exc() - print("thread %s: %s dumps, loads ERROR" % (get_ident(), n)) + print(f"thread {get_ident()}: {n} dumps, loads ERROR") with ThreadPoolExecutor(max_workers=4) as executor: - executor.map(test_func, range(200000), chunksize=1000) + executor.map(test_func, range(50000), chunksize=1000) executor.shutdown(wait=True) diff --git a/integration/typestubs.py b/integration/typestubs.py index e7527e99..53776113 100644 --- a/integration/typestubs.py +++ b/integration/typestubs.py @@ -1,5 +1,8 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright Eric Jolibois (2022), ijl (2023) import orjson orjson.JSONDecodeError(msg="the_msg", doc="the_doc", pos=1) + +orjson.dumps(orjson.Fragment(b"{}")) diff --git a/integration/wsgi.py b/integration/wsgi.py index 9418aafd..846c63b8 100644 --- a/integration/wsgi.py +++ b/integration/wsgi.py @@ -1,22 +1,31 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2018-2025) -import os -import lzma +from datetime import datetime, timezone +from uuid import uuid4 from flask import Flask + import orjson app = Flask(__name__) -filename = os.path.join(os.path.dirname(__file__), "..", "data", "twitter.json.xz") - -with lzma.open(filename, "r") as fileh: - DATA = orjson.loads(fileh.read()) +NOW = datetime.now(timezone.utc) @app.route("/") def root(): - data = orjson.dumps(DATA) + data = { + "uuid": uuid4(), + "updated_at": NOW, + "data": [1, 2.2, None, True, False, orjson.Fragment(b"{}")], + } + payload = orjson.dumps( + data, + option=orjson.OPT_NAIVE_UTC | orjson.OPT_OMIT_MICROSECONDS, + ) return app.response_class( - response=data, status=200, mimetype="application/json; charset=utf-8" + response=payload, + status=200, + mimetype="application/json; charset=utf-8", ) diff --git a/pyproject.toml b/pyproject.toml index 335cd604..b4624c62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,16 +1,103 @@ [project] name = "orjson" +version = "3.11.9" repository = "https://github.com/ijl/orjson" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +requires-python = ">=3.10" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "License :: OSI Approved :: MIT License", + "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python", + "Programming Language :: Rust", + "Typing :: Typed", +] +readme = "README.md" +license = "MPL-2.0 AND (Apache-2.0 OR MIT)" + +[project.urls] +source = "https://github.com/ijl/orjson" +documentation = "https://github.com/ijl/orjson" +changelog = "https://github.com/ijl/orjson/blob/master/CHANGELOG.md" [build-system] build-backend = "maturin" -requires = ["maturin>=0.12.19,<0.13"] +requires = ["maturin>=1,<2"] [tool.maturin] -sdist-include = ["build.rs", "Cargo.lock", "include"] -strip = true +python-source = "pysrc" +include = [ + { format = "sdist", path = ".cargo/*" }, + { format = "sdist", path = "build.rs" }, + { format = "sdist", path = "Cargo.lock" }, + { format = "sdist", path = "include/cargo/**/*" }, + { format = "sdist", path = "include/yyjson/**/*" }, +] -[tool.black] +[tool.ruff] line-length = 88 -target-version = ["py37"] -include = ".pyi?$" +target-version = "py310" + +[tool.ruff.lint] +select = [ + "A", + "ASYNC", + "B", + "COM", + "DTZ", + "E", + "EXE", + "F", + "FLY", + "I", + "ISC", + "PIE", + "PLC", + "PLE", + "PLR", + "PLW", + "RUF", + "TCH", + "TID", + "UP", + "W", +] +ignore = [ + "B005", # Using `.strip()` with multi-character strings is misleading + "DTZ001", # `datetime.datetime()` called without a `tzinfo` argument + "DTZ005", # `datetime.datetime.now()` called without a `tz` argument + "E402", # Module level import not at top of file + "E501", # line too long + "F601", # Dictionary key literal ... repeated + "PIE810", # Call `startswith` once with a `tuple` + "PLR2004", # Magic value used in comparison + "RUF061", # Use context-manager form of `pytest.raises()` + "UP012", # Unnecessary call to encode as UTF-8 +] + +[tool.pytest] +minversion = "9.0" +strict = true + +[tool.ruff.lint.isort] +known-first-party = ["orjson"] + +[tool.mypy] +python_version = "3.10" + +[[tool.mypy.overrides]] +module = ["dateutil", "pytz"] +ignore_missing_imports = true diff --git a/pysrc/orjson/__init__.py b/pysrc/orjson/__init__.py new file mode 100644 index 00000000..c6a12ab6 --- /dev/null +++ b/pysrc/orjson/__init__.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2023-2026) + +""" +Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy +""" + +from .orjson import * +from .orjson import __version__ + +__all__ = ( + "__version__", + "dumps", + "Fragment", + "JSONDecodeError", + "JSONEncodeError", + "loads", + "OPT_APPEND_NEWLINE", + "OPT_INDENT_2", + "OPT_NAIVE_UTC", + "OPT_NON_STR_KEYS", + "OPT_OMIT_MICROSECONDS", + "OPT_PASSTHROUGH_DATACLASS", + "OPT_PASSTHROUGH_DATETIME", + "OPT_PASSTHROUGH_SUBCLASS", + "OPT_SERIALIZE_DATACLASS", + "OPT_SERIALIZE_NUMPY", + "OPT_SERIALIZE_UUID", + "OPT_SORT_KEYS", + "OPT_STRICT_INTEGER", + "OPT_UTC_Z", +) diff --git a/orjson.pyi b/pysrc/orjson/__init__.pyi similarity index 57% rename from orjson.pyi rename to pysrc/orjson/__init__.pyi index 5e5d697b..9804cee5 100644 --- a/orjson.pyi +++ b/pysrc/orjson/__init__.pyi @@ -1,18 +1,25 @@ +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2019-2026), Eric Jolibois (2022), Anders Kaseorg (2020) + import json -from typing import Any, Callable, Optional, Union +from collections.abc import Callable +from typing import Any __version__: str def dumps( __obj: Any, - default: Optional[Callable[[Any], Any]] = ..., - option: Optional[int] = ..., + default: Callable[[Any], Any] | None = ..., + option: int | None = ..., ) -> bytes: ... -def loads(__obj: Union[bytes, bytearray, memoryview, str]) -> Any: ... +def loads(__obj: bytes | bytearray | memoryview | str) -> Any: ... class JSONDecodeError(json.JSONDecodeError): ... class JSONEncodeError(TypeError): ... +class Fragment(tuple): + contents: bytes | str + OPT_APPEND_NEWLINE: int OPT_INDENT_2: int OPT_NAIVE_UTC: int diff --git a/pysrc/orjson/py.typed b/pysrc/orjson/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/requirements-lint.txt b/requirements-lint.txt new file mode 100644 index 00000000..c97b9884 --- /dev/null +++ b/requirements-lint.txt @@ -0,0 +1,2 @@ +mypy==1.19.1;python_version<"3.15" +ruff==0.15.8 diff --git a/requirements.txt b/requirements.txt index e93c377e..54ef6cbe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,5 @@ -r bench/requirements.txt -r integration/requirements.txt +-r requirements-lint.txt -r test/requirements.txt -autoflake -black -isort -maturin -mypy -types-python-dateutil -types-pytz -types-simplejson -types-ujson +maturin>=1.10,<2 diff --git a/script/blame-to-header b/script/blame-to-header new file mode 100755 index 00000000..679a2fda --- /dev/null +++ b/script/blame-to-header @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2026) + +import asyncio +import datetime +import re +import subprocess +from collections import defaultdict +from pathlib import Path + +REPOSITORY = Path(__file__).parent.parent + +LICENSE_APACHE = "(Apache-2.0 OR MIT)" +LICENSE_MPL2 = "MPL-2.0" + +SPACES = re.compile(r"[ ]{2,}") + +TO_INCLUDE = { + ".github/**/*.yaml", + "bench/*.py", + "bench/run", + "integration/*", + "pysrc/orjson/*", + "script/*", + "src/**/*.rs", + "test/**/*.py", +} + +TO_EXCLUDE = { + "integration/http", + "script/cargo", + "script/debug", + "script/develop", + "script/install-fedora", + "script/lint", + "script/profile", + "script/pybench", + "script/pytest", + "script/valgrind", + "src/serialize/writer/half.rs", + "src/serialize/writer/uuid.rs", +} + + +def aggregate_files() -> list[Path]: + files = [] + + files.append(REPOSITORY / Path("build.rs")) + + for pattern in TO_INCLUDE: + files.extend(REPOSITORY.glob(pattern)) + + files = { + each + for each in files + if not str(each).endswith(("py.typed", "__pycache__", ".txt")) + } + + for filename in TO_EXCLUDE: + files.remove(REPOSITORY / Path(filename)) + + return sorted(list(files)) + + +SKIP_FRAGMENTS = ( + "# Copyright", + "# SPDX", + "#!/usr", + "// Copyright", + "// SPDX", +) + + +def get_contributor_and_date(line: str) -> tuple[str, datetime.date] | None: + if not line or "Not Committed Yet" in line: + return None + + end = line.index(")") + diff = line[end + 2 :] + diff = SPACES.sub(r" ", diff) + + # skip blank and ' };' etc + if len(diff) <= 3: + return None + + # skip headers and imports + for fragment in SKIP_FRAGMENTS: + if diff.startswith(fragment): + return None + + line = line[line.index("(") + 1 : end] + line = SPACES.sub(r" ", line) + segments = line.split(" ")[0:-2] + contributor = " ".join(segments[:-1]) + date = datetime.date.fromtimestamp(int(segments[-1])).year + return (contributor, date) + + +def process_blame(filename: Path, blame: str) -> list[str, list[str]] | None: + file_license = "(Apache-2.0 OR MIT)" + contributors = defaultdict(list) + document = blame.split("\n") + for line in document: + ret = get_contributor_and_date(line) + if ret: + contributors[ret[0]].append(ret[1]) + + overall_earliest = 9999 + overall_latest = 0 + file_credit = [] + for contributor, dates in contributors.items(): + earliest = min(dates) + latest = max(dates) + overall_latest = max((latest, overall_latest)) + overall_earliest = min((latest, overall_earliest)) + num_lines = len(dates) + if earliest == latest: + file_credit.append((num_lines, f"{contributor} ({earliest})")) + else: + file_credit.append((num_lines, f"{contributor} ({earliest}-{latest})")) + + if (len(contributors) == 1 and "ijl" in contributors) or overall_earliest == 2026: + file_license = LICENSE_MPL2 + + file_credit.sort(reverse=True) + + return [file_license, file_credit] + + +async def handle_file(filename: str): + blame = await asyncio.create_subprocess_shell( + f"git blame -C -C -C -M --date=raw {filename}", + shell=True, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + header = process_blame(filename, (await blame.stdout.read()).decode("utf-8")) + if header is None: + print(f"{filename.relative_to(REPOSITORY)} skipping") + return + if not header[1] and str(filename).endswith("__init__.py"): + return + + if str(filename).endswith(".rs"): + prefix = "//" + else: + prefix = "#" + + spdx = f"{prefix} SPDX-License-Identifier: {header[0]}" + credit = f"{prefix} Copyright {', '.join(each[1] for each in header[1])}" + + contents = filename.read_bytes().decode("utf-8").split("\n") + if contents[0].startswith("#!"): + start_idx = 1 + else: + start_idx = 0 + + if contents[start_idx].startswith(f"{prefix} SPDX-License-Identifier"): + contents[start_idx] = spdx + else: + contents.insert(start_idx, spdx) + + if contents[start_idx + 1].startswith(f"{prefix} Copyright"): + contents[start_idx + 1] = credit + else: + contents.insert(start_idx + 1, credit) + + # separate by blank line + first_line = start_idx + 2 + while contents[first_line].startswith(prefix): + first_line += 1 + if len(contents[first_line]) > 0: + contents.insert(first_line, "") + + print(f"{filename.relative_to(REPOSITORY)} {spdx}") + + filename.write_bytes("\n".join(contents).encode("utf-8")) + + +async def main(): + async with asyncio.TaskGroup() as tg: + for filename in aggregate_files(): + tg.create_task(handle_file(filename)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/script/cargo b/script/cargo new file mode 100755 index 00000000..f27dea68 --- /dev/null +++ b/script/cargo @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -eou pipefail + +export UNSAFE_PYO3_BUILD_FREE_THREADED=1 +export UNSAFE_PYO3_SKIP_VERSION_CHECK=1 + +RUSTFLAGS="-Z unstable-options -C panic=immediate-abort -Z panic_abort_tests" cargo "$@" --target="${TARGET:-x86_64-unknown-linux-gnu}" diff --git a/script/check-pypi b/script/check-pypi new file mode 100755 index 00000000..c4c3ce75 --- /dev/null +++ b/script/check-pypi @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2025) + +import sys +from pathlib import Path + +import tomllib + +dist = sys.argv[1] + +pyproject_doc = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) +pyproject_version = pyproject_doc["project"]["version"] + +prefix = f"orjson-{pyproject_version}" + +abis = ( + "cp310-cp310", + "cp311-cp311", + "cp312-cp312", + "cp313-cp313", + "cp314-cp314", +) + +per_abi_tags = ( + "macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2", + "manylinux_2_17_aarch64.manylinux2014_aarch64", + "manylinux_2_17_armv7l.manylinux2014_armv7l", + "manylinux_2_17_ppc64le.manylinux2014_ppc64le", + "manylinux_2_17_s390x.manylinux2014_s390x", + "manylinux_2_17_x86_64.manylinux2014_x86_64", + "manylinux_2_17_i686.manylinux2014_i686", + "musllinux_1_2_aarch64", + "musllinux_1_2_armv7l", + "musllinux_1_2_i686", + "musllinux_1_2_x86_64", + "win32", + "win_amd64", +) + +wheels_matrix = set() + +# orjson-3.10.15-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl +for abi in abis: + for tag in per_abi_tags: + wheels_matrix.add(f"{prefix}-{abi}-{tag}.whl") + +wheels_prerelease = set() + +wheels_unique = { + f"{prefix}-cp311-cp311-macosx_15_0_arm64.whl", + f"{prefix}-cp311-cp311-win_arm64.whl", + f"{prefix}-cp312-cp312-macosx_15_0_arm64.whl", + f"{prefix}-cp312-cp312-win_arm64.whl", + f"{prefix}-cp313-cp313-macosx_15_0_arm64.whl", + f"{prefix}-cp313-cp313-win_arm64.whl", + f"{prefix}-cp314-cp314-macosx_15_0_arm64.whl", + f"{prefix}-cp314-cp314-win_arm64.whl", +} + +wheels_expected = wheels_matrix | wheels_unique | wheels_prerelease + +wheels_queued = set( + str(each).replace(f"{dist}/", "") for each in Path(dist).glob("*.whl") +) + +exit_code = 0 + +# sdist +sdist_path = Path(f"{dist}/{prefix}.tar.gz") +if sdist_path.exists(): + print("sdist present\n") +else: + exit_code = 1 + print(f"Missing sdist:\n{sdist_path}\n") + +# whl +if wheels_expected == wheels_queued: + print(f"Wheels as expected, {len(wheels_queued)} total\n") +else: + exit_code = 1 + + missing = "\n".join(sorted(wheels_expected - wheels_queued)) + if missing: + print(f"Missing wheels:\n{missing}\n") + + additional = "\n".join(sorted(wheels_queued - wheels_expected)) + if additional: + print(f"Unexpected wheels:\n{additional}\n") + +sys.exit(exit_code) diff --git a/script/check-version b/script/check-version new file mode 100755 index 00000000..972b1419 --- /dev/null +++ b/script/check-version @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2025) + +from pathlib import Path + +import tomllib + +cargo_doc = tomllib.loads(Path("Cargo.toml").read_text(encoding="utf-8")) +pyproject_doc = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) + +cargo_version = cargo_doc["package"]["version"] +pyproject_version = pyproject_doc["project"]["version"] + +print(f"Cargo.toml version: {cargo_version}") +print(f"pyproject.toml version: {pyproject_version}") + +assert cargo_version == pyproject_version diff --git a/script/debug b/script/debug new file mode 100755 index 00000000..6833bf6b --- /dev/null +++ b/script/debug @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -eou pipefail + +rm -rf .cargo +rm -f ${CARGO_TARGET_DIR}/wheels/*.whl + +export UNSAFE_PYO3_BUILD_FREE_THREADED=1 +export UNSAFE_PYO3_SKIP_VERSION_CHECK=1 + +export CC="${CC:-clang}" +export LD="${LD:-lld}" +export TARGET="${TARGET:-x86_64-unknown-linux-gnu}" +export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-target}" + +export CFLAGS="-Os -fstrict-aliasing" + +export RUSTFLAGS="-C panic=unwind -C linker=${CC} -C link-arg=-fuse-ld=${LD}" + +maturin build --profile=dev --target="${TARGET}" + +uv pip install ${CARGO_TARGET_DIR}/wheels/*.whl + +pytest -v test + +mkdir .cargo +cp ci/config.toml .cargo diff --git a/script/develop b/script/develop index 93a5fce9..a0813683 100755 --- a/script/develop +++ b/script/develop @@ -2,11 +2,26 @@ rm -f target/wheels/* -export CC="clang" -export CFLAGS="-O2 -fno-plt -flto=thin" -export LDFLAGS="-O2 -flto=thin -fuse-ld=lld -Wl,--as-needed" -export RUSTFLAGS="-C linker=clang -C link-arg=-fuse-ld=lld" +export UNSAFE_PYO3_BUILD_FREE_THREADED=1 +export UNSAFE_PYO3_SKIP_VERSION_CHECK=1 -maturin build --no-sdist --compatibility off -i python3 --release "$@" +mkdir -p .cargo +cp ci/config.toml .cargo/config.toml -pip install --force $(find target/wheels -name "*cp3*") +export CC="${CC:-clang}" +export LD="${LD:-lld}" +export TARGET="${TARGET:-x86_64-unknown-linux-gnu}" +export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-target}" +export ORJSON_COMPATIBILITY="${ORJSON_COMPATIBILITY:-manylinux_2_17}" + +echo "CC: ${CC}, LD: ${LD}, LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}" + +export CFLAGS="-O2 -fstrict-aliasing -fno-plt -emit-llvm" +export LDFLAGS="-fuse-ld=${LD} -Wl,-plugin-opt=also-emit-llvm -Wl,--as-needed -Wl,-zrelro,-znow" +export RUSTFLAGS="-Z unstable-options -C panic=immediate-abort -C linker=${CC} -C link-arg=-fuse-ld=${LD} -C linker-plugin-lto -C link-arg=-Wl,-zrelro,-znow -Z mir-opt-level=4 -Z threads=8" + +rm -f ${CARGO_TARGET_DIR}/wheels/*.whl + +maturin build --target="${TARGET}" --features=no_panic,optimize --compatibility="${ORJSON_COMPATIBILITY}" --release + +uv pip install --link-mode=copy ${CARGO_TARGET_DIR}/wheels/*.whl diff --git a/script/generate-yyjson b/script/generate-yyjson deleted file mode 100755 index 819df132..00000000 --- a/script/generate-yyjson +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -_repo="$(dirname "$(dirname "${BASH_SOURCE[0]}")")" - -bindgen \ - "${_repo}/include/yyjson/yyjson.h" \ - --size_t-is-usize \ - --disable-header-comment \ - --no-derive-debug \ - --no-doc-comments \ - --no-layout-tests \ - --allowlist-function=yyjson_alc_pool_init \ - --allowlist-function=yyjson_doc_free \ - --allowlist-function=yyjson_read_opts \ - --allowlist-type=yyjson_alc \ - --allowlist-type=yyjson_arr_iter \ - --allowlist-type=yyjson_doc \ - --allowlist-type=yyjson_obj_iter \ - --allowlist-type=yyjson_read_code \ - --allowlist-type=yyjson_read_err \ - --allowlist-type=yyjson_val \ - --allowlist-var=YYJSON_READ_NOFLAG \ - --allowlist-var=YYJSON_READ_SUCCESS \ - > "${_repo}/src/yyjson.rs" diff --git a/script/graph b/script/graph index 89c86303..e7b335fb 100755 --- a/script/graph +++ b/script/graph @@ -1,22 +1,26 @@ #!/usr/bin/env python3 -# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2018-2025) import collections import io import os +import pandas as pd +import seaborn as sns +from matplotlib import pyplot as plt from tabulate import tabulate import orjson -LIBRARIES = ("orjson", "ujson", "rapidjson", "simplejson", "json") +LIBRARIES = ("orjson", "json") def aggregate(): benchmarks_dir = os.path.join(".benchmarks", os.listdir(".benchmarks")[0]) res = collections.defaultdict(dict) for filename in os.listdir(benchmarks_dir): - with open(os.path.join(benchmarks_dir, filename), "r") as fileh: + with open(os.path.join(benchmarks_dir, filename)) as fileh: data = orjson.loads(fileh.read()) for each in data["benchmarks"]: @@ -37,6 +41,11 @@ def tab(obj): "Operations per second", "Relative (latency)", ) + + sns.set(rc={"figure.facecolor": (0, 0, 0, 0)}) + sns.set_style("darkgrid") + + barplot_data = [] for group, val in sorted(obj.items(), reverse=True): buf.write("\n" + "#### " + group + "\n\n") table = [] @@ -46,18 +55,93 @@ def tab(obj): [ lib, val[lib]["median"] if correct else None, - "%.1f" % val[lib]["ops"] if correct else None, + int(val[lib]["ops"]) if correct else None, 0, - ] + ], ) - baseline = table[0][1] + barplot_data.append( + { + "operation": "deserialization" + if "deserialization" in group + else "serialization", + "group": group.strip("serialization") + .strip("deserialization") + .strip(), + "library": lib, + "latency": val[lib]["median"], + "operations": int(val[lib]["ops"]) if correct else None, + }, + ) + + orjson_baseline = table[0][1] for each in table: each[3] = ( - "%.2f" % (each[1] / baseline) if isinstance(each[1], float) else None + "%.1f" % (each[1] / orjson_baseline) + if isinstance(each[1], float) + else None ) - each[1] = "%.2f" % each[1] if isinstance(each[1], float) else None + if group.startswith("github"): + each[1] = f"{each[1]:.2f}" if isinstance(each[1], float) else None + else: + each[1] = f"{each[1]:.1f}" if isinstance(each[1], float) else None + buf.write(tabulate(table, headers, tablefmt="github") + "\n") + for operation in ("deserialization", "serialization"): + per_op_data = list( + each for each in barplot_data if each["operation"] == operation + ) + if not per_op_data: + continue + + max_y = 10 if operation == "serialization" else 5 + + json_baseline = {} + for each in per_op_data: + if each["group"] == "witter.json": + each["group"] = "twitter.json" + if each["library"] == "json": + json_baseline[each["group"]] = each["operations"] + + for each in per_op_data: + relative = each["operations"] / json_baseline[each["group"]] + each["relative"] = min(max_y, relative) + + p = pd.DataFrame.from_dict(per_op_data) + p.groupby("group") + + graph = sns.barplot( + p, + x="group", + y="relative", + orient="x", + hue="library", + errorbar="sd", + legend="brief", + ) + graph.set_xlabel("Document") + graph.set_ylabel("Operations/second relative to stdlib json") + + plt.title(operation) + + # ensure Y range + plt.gca().set_yticks( + list( + {1, max_y}.union( + set(int(y) for y in plt.gca().get_yticks() if int(y) <= max_y), + ), + ), + ) + + # print Y as percent + plt.gca().set_yticklabels([f"{x}x" for x in plt.gca().get_yticks()]) + + # reference for stdlib + plt.axhline(y=1, color="#999", linestyle="dashed") + + plt.savefig(fname=f"doc/{operation}", dpi=300) + plt.close() + print(buf.getvalue()) diff --git a/script/install-fedora b/script/install-fedora new file mode 100755 index 00000000..fdc58e7b --- /dev/null +++ b/script/install-fedora @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -eou pipefail + +export VENV="${VENV:-.venv}" +export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-target}" + +rm -f /etc/yum.repos.d/fedora-cisco-openh264.repo || true +echo 'max_parallel_downloads=10' >> /etc/dnf/dnf.conf +echo 'timeout=1' >> /etc/dnf/dnf.conf + +packages="rustup clang lld "${PYTHON_PACKAGE}" python3-uv" + +if [[ $# -gt 1 ]]; then + packages="${packages} $@" +fi + + +dnf install --setopt=install_weak_deps=false -y $packages + +rustup-init --default-toolchain "${RUST_TOOLCHAIN}-${TARGET}" --profile minimal --component rust-src -y +source "${HOME}/.cargo/env" + +mkdir -p .cargo +cp ci/config.toml .cargo/config.toml + +cargo fetch --target="${TARGET}" & + +rm -rf "${VENV}" +uv venv --python "${PYTHON}" "${VENV}" +source "${VENV}/bin/activate" + +uv pip install --upgrade "maturin>=1.10,<2" -r test/requirements.txt -r integration/requirements.txt diff --git a/script/lint b/script/lint index cd55e722..600fa5b4 100755 --- a/script/lint +++ b/script/lint @@ -2,10 +2,11 @@ set -eou pipefail -to_lint="./bench/*.py ./orjson.pyi ./test/*.py script/pydataclass script/pymem -script/pysort script/pynumpy script/pynonstr script/pycorrectness script/graph" +to_lint="./bench/*.py ./pysrc/orjson/__init__.pyi ./test/*.py script/pydataclass +script/pysort script/pynumpy script/pynonstr script/pycorrectness script/graph integration/init +integration/wsgi.py integration/typestubs.py integration/thread script/check-version +script/check-pypi" -autoflake --in-place --recursive --remove-all-unused-imports --ignore-init-module-imports . -isort ${to_lint} -black ${to_lint} -mypy --ignore-missing-imports ./bench/*.py ./orjson.pyi ./test/*.py +ruff format ${to_lint} +ruff check ${to_lint} --fix +mypy --ignore-missing-imports --check-untyped-defs ./bench/*.py ./pysrc/orjson/__init__.pyi ./test/*.py diff --git a/script/profile b/script/profile index 47be066d..ea94fd4f 100755 --- a/script/profile +++ b/script/profile @@ -4,4 +4,3 @@ perf record -g --delay 250 ./bench/run_func "$@" perf report --percent-limit 0.1 -rm -f perf.data* diff --git a/script/pybench-empty b/script/pybench-empty deleted file mode 100755 index 1e53454d..00000000 --- a/script/pybench-empty +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -pytest \ - --verbose \ - --benchmark-min-time=1 \ - --benchmark-max-time=5 \ - --benchmark-disable-gc \ - --benchmark-autosave \ - --benchmark-save-data \ - --random-order \ - -k orjson \ - "bench/benchmark_empty.py" diff --git a/script/pycorrectness b/script/pycorrectness index 9ef6984c..6945c545 100755 --- a/script/pycorrectness +++ b/script/pycorrectness @@ -1,8 +1,10 @@ #!/usr/bin/env python3 -# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2019-2025) import collections import io +import json import lzma import os from pathlib import Path @@ -11,9 +13,15 @@ from tabulate import tabulate import orjson -dirname = os.path.join(os.path.dirname(__file__), "data") +dirname = os.path.join(os.path.dirname(__file__), "..", "data") -LIBRARIES = ["orjson", "ujson", "rapidjson", "simplejson", "json"] +LIBRARIES = ["orjson", "json"] + + +LIBRARY_FUNC_MAP = { + "orjson": orjson.loads, + "json": json.loads, +} def read_fixture_bytes(filename, subdir=None): @@ -43,35 +51,30 @@ JSONCHECKER = { RESULTS = collections.defaultdict(dict) -def read_fixture(filename, subdir=None): - if not filename in BYTES_CACHE: - BYTES_CACHE[filename] = read_fixture_bytes(filename, subdir) - return BYTES_CACHE[filename] - - -def test_passed(library, fixture): +def test_passed(libname, fixture): passed = [] + loads = LIBRARY_FUNC_MAP[libname] try: - passed.append(library.loads(fixture) == orjson.loads(fixture)) + passed.append(loads(fixture) == orjson.loads(fixture)) passed.append( - library.loads(fixture.decode("utf-8")) - == orjson.loads(fixture.decode("utf-8")) + loads(fixture.decode("utf-8")) == orjson.loads(fixture.decode("utf-8")), ) except Exception: passed.append(False) return all(passed) -def test_failed(library, fixture): +def test_failed(libname, fixture): rejected_as_bytes = False + loads = LIBRARY_FUNC_MAP[libname] try: - library.loads(fixture) + loads(fixture) except Exception: rejected_as_bytes = True rejected_as_str = False try: - library.loads(fixture.decode("utf-8")) + loads(fixture.decode("utf-8")) except Exception: rejected_as_str = True return rejected_as_bytes and rejected_as_str @@ -98,21 +101,20 @@ def should_fail(filename): or filename.startswith("i_string") or filename.startswith("i_object") or filename.startswith("fail") - ) and not filename in PASS_WHITELIST + ) and filename not in PASS_WHITELIST for libname in LIBRARIES: - library = __import__(libname) for fixture_set in (PARSING, JSONCHECKER): for filename, fixture in fixture_set.items(): if should_pass(filename): - res = test_passed(library, fixture) + res = test_passed(libname, fixture) RESULTS[filename][libname] = res if not res: MISTAKEN_PASSES[libname] += 1 elif should_fail(filename): - res = test_failed(library, fixture) + res = test_failed(libname, fixture) RESULTS[filename][libname] = res if not res: MISTAKEN_FAILS[libname] += 1 @@ -137,7 +139,7 @@ for filename in FILENAMES: tab_results.append(entry) buf = io.StringIO() -buf.write(tabulate(tab_results, ["Fixture"] + LIBRARIES, tablefmt="github")) +buf.write(tabulate(tab_results, ["Fixture", *LIBRARIES], tablefmt="github")) buf.write("\n") print(buf.getvalue()) @@ -156,7 +158,7 @@ buf.write( "Valid JSON documents not deserialized", ], tablefmt="github", - ) + ), ) buf.write("\n") print(buf.getvalue()) diff --git a/script/pydataclass b/script/pydataclass index b43c95a4..1c4245f1 100755 --- a/script/pydataclass +++ b/script/pydataclass @@ -1,21 +1,19 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2019-2025), Aarni Koskela (2021) import dataclasses import io import json import os from timeit import timeit -from typing import List -import rapidjson -import simplejson -import ujson from tabulate import tabulate import orjson -os.sched_setaffinity(os.getpid(), {0, 1}) +if hasattr(os, "sched_setaffinity"): + os.sched_setaffinity(os.getpid(), {0, 1}) @dataclasses.dataclass @@ -28,11 +26,11 @@ class Member: class Object: id: int name: str - members: List[Member] + members: list[Member] objects_as_dataclass = [ - Object(i, str(i) * 3, [Member(j, True) for j in range(0, 10)]) + Object(i, str(i) * 3, [Member(j, True) for j in range(10)]) for i in range(100000, 102000) ] @@ -50,7 +48,7 @@ def default(__obj): headers = ("Library", "dict (ms)", "dataclass (ms)", "vs. orjson") -LIBRARIES = ("orjson", "ujson", "rapidjson", "simplejson", "json") +LIBRARIES = ("orjson", "json") ITERATIONS = 100 @@ -72,39 +70,13 @@ for lib_name in LIBRARIES: lambda: json.dumps(objects_as_dataclass, default=default).encode("utf-8"), number=ITERATIONS, ) - elif lib_name == "simplejson": - as_dict = timeit( - lambda: simplejson.dumps(objects_as_dict).encode("utf-8"), - number=ITERATIONS, - ) - as_dataclass = timeit( - lambda: simplejson.dumps(objects_as_dataclass, default=default).encode( - "utf-8" - ), - number=ITERATIONS, - ) - elif lib_name == "ujson": - as_dict = timeit( - lambda: ujson.dumps(objects_as_dict).encode("utf-8"), - number=ITERATIONS, - ) - as_dataclass = None - elif lib_name == "rapidjson": - as_dict = timeit( - lambda: rapidjson.dumps(objects_as_dict).encode("utf-8"), - number=ITERATIONS, - ) - as_dataclass = timeit( - lambda: rapidjson.dumps(objects_as_dataclass, default=default).encode( - "utf-8" - ), - number=ITERATIONS, - ) elif lib_name == "orjson": as_dict = timeit(lambda: orjson.dumps(objects_as_dict), number=ITERATIONS) as_dataclass = timeit( lambda: orjson.dumps( - objects_as_dataclass, None, orjson.OPT_SERIALIZE_DATACLASS + objects_as_dataclass, + None, + orjson.OPT_SERIALIZE_DATACLASS, ), number=ITERATIONS, ) @@ -128,7 +100,7 @@ for lib_name in LIBRARIES: f"{as_dict:,.2f}" if as_dict else "", f"{as_dataclass:,.2f}" if as_dataclass else "", f"{compared_to_orjson:d}" if compared_to_orjson else "", - ) + ), ) buf = io.StringIO() diff --git a/script/pyindent b/script/pyindent index da576ec8..69f74a5c 100755 --- a/script/pyindent +++ b/script/pyindent @@ -1,5 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2019-2024), Aarni Koskela (2021) import io import json @@ -9,17 +10,15 @@ import sys from pathlib import Path from timeit import timeit -import rapidjson -import simplejson -import ujson from tabulate import tabulate import orjson -os.sched_setaffinity(os.getpid(), {0, 1}) +if hasattr(os, "sched_setaffinity"): + os.sched_setaffinity(os.getpid(), {0, 1}) -dirname = os.path.join(os.path.dirname(__file__), "data") +dirname = os.path.join(os.path.dirname(__file__), "..", "data") def read_fixture_obj(filename): @@ -37,7 +36,7 @@ data = read_fixture_obj(f"{filename}.json.xz") headers = ("Library", "compact (ms)", "pretty (ms)", "vs. orjson") -LIBRARIES = ("orjson", "ujson", "rapidjson", "simplejson", "json") +LIBRARIES = ("orjson", "json") output_in_kib_compact = len(orjson.dumps(data)) / 1024 output_in_kib_pretty = len(orjson.dumps(data, option=orjson.OPT_INDENT_2)) / 1024 @@ -73,30 +72,6 @@ for lib_name in LIBRARIES: number=ITERATIONS, ) correct = test_correctness(json.dumps(data, indent=2).encode("utf-8")) - elif lib_name == "simplejson": - time_compact = timeit( - lambda: simplejson.dumps(data).encode("utf-8"), - number=ITERATIONS, - ) - time_pretty = timeit( - lambda: simplejson.dumps(data, indent=2).encode("utf-8"), - number=ITERATIONS, - ) - correct = test_correctness(simplejson.dumps(data, indent=2).encode("utf-8")) - elif lib_name == "ujson": - time_compact = timeit( - lambda: ujson.dumps(data).encode("utf-8"), - number=ITERATIONS, - ) - time_pretty = timeit( - lambda: ujson.dumps(data, indent=2).encode("utf-8"), - number=ITERATIONS, - ) - correct = test_correctness(ujson.dumps(data, indent=2).encode("utf-8")) - elif lib_name == "rapidjson": - time_compact = timeit(lambda: rapidjson.dumps(data), number=ITERATIONS) - time_pretty = None - correct = False elif lib_name == "orjson": time_compact = timeit(lambda: orjson.dumps(data), number=ITERATIONS) time_pretty = timeit( diff --git a/script/pymem b/script/pymem deleted file mode 100755 index 6b97bb0d..00000000 --- a/script/pymem +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import io -import subprocess - -from tabulate import tabulate - -buf = io.StringIO() - -headers = ("Library", "import, read() RSS (MiB)", "loads() increase in RSS (MiB)") - -LIBRARIES = ("orjson", "ujson", "rapidjson", "simplejson", "json") - -FIXTURES = ("canada.json", "citm_catalog.json", "github.json", "twitter.json") - -for fixture in sorted(FIXTURES, reverse=True): - table = [] - buf.write("\n" + "#### " + fixture + "\n\n") - for lib_name in LIBRARIES: - proc = subprocess.Popen( - ("bench/run_mem", f"data/{fixture}.xz", lib_name), stdout=subprocess.PIPE - ) - output = proc.stdout.readline().decode("utf-8").strip().split(",") - mem_base = int(output[0]) / 1024 / 1024 - mem_diff = int(output[1]) / 1024 / 1024 - correct = bool(int(output[2])) - if correct: - table.append((lib_name, f"{mem_base:,.1f}", f"{mem_diff:,.1f}")) - else: - table.append((lib_name, "", "")) - buf.write(tabulate(table, headers, tablefmt="github") + "\n") - -print(buf.getvalue()) diff --git a/script/pynonstr b/script/pynonstr index 9670443a..bf3ba400 100755 --- a/script/pynonstr +++ b/script/pynonstr @@ -1,5 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2020-2025), Aarni Koskela (2021) import datetime import io @@ -9,21 +10,19 @@ import random from time import mktime from timeit import timeit -import rapidjson -import simplejson -import ujson from tabulate import tabulate import orjson -os.sched_setaffinity(os.getpid(), {0, 1}) +if hasattr(os, "sched_setaffinity"): + os.sched_setaffinity(os.getpid(), {0, 1}) data_as_obj = [] for year in range(1920, 2020): start = datetime.date(year, 1, 1) array = [ (int(mktime((start + datetime.timedelta(days=i)).timetuple())), i + 1) - for i in range(0, 365) + for i in range(365) ] array.append(("other", 0)) random.shuffle(array) @@ -33,7 +32,7 @@ data_as_str = orjson.loads(orjson.dumps(data_as_obj, option=orjson.OPT_NON_STR_K headers = ("Library", "str keys (ms)", "int keys (ms)", "int keys sorted (ms)") -LIBRARIES = ("orjson", "ujson", "rapidjson", "simplejson", "json") +LIBRARIES = ("orjson", "json") ITERATIONS = 500 @@ -69,41 +68,6 @@ for lib_name in LIBRARIES: None # TypeError: '<' not supported between instances of 'str' and 'int' ) correct = False - elif lib_name == "simplejson": - time_as_str = timeit( - lambda: simplejson.dumps(data_as_str).encode("utf-8"), - number=ITERATIONS, - ) - time_as_obj = timeit( - lambda: simplejson.dumps(data_as_obj).encode("utf-8"), - number=ITERATIONS, - ) - time_as_obj_sorted = timeit( - lambda: simplejson.dumps(data_as_obj, sort_keys=True).encode("utf-8"), - number=ITERATIONS, - ) - correct = test_correctness( - simplejson.dumps(data_as_obj, sort_keys=True).encode("utf-8") - ) - elif lib_name == "ujson": - time_as_str = timeit( - lambda: ujson.dumps(data_as_str).encode("utf-8"), - number=ITERATIONS, - ) - time_as_obj = timeit( - lambda: ujson.dumps(data_as_obj).encode("utf-8"), - number=ITERATIONS, - ) - time_as_obj_sorted = None # segfault - correct = False - elif lib_name == "rapidjson": - time_as_str = timeit( - lambda: rapidjson.dumps(data_as_str).encode("utf-8"), - number=ITERATIONS, - ) - time_as_obj = None - time_as_obj_sorted = None - correct = False elif lib_name == "orjson": time_as_str = timeit( lambda: orjson.dumps(data_as_str, None, orjson.OPT_NON_STR_KEYS), @@ -115,14 +79,18 @@ for lib_name in LIBRARIES: ) time_as_obj_sorted = timeit( lambda: orjson.dumps( - data_as_obj, None, orjson.OPT_NON_STR_KEYS | orjson.OPT_SORT_KEYS + data_as_obj, + None, + orjson.OPT_NON_STR_KEYS | orjson.OPT_SORT_KEYS, ), number=ITERATIONS, ) correct = test_correctness( orjson.dumps( - data_as_obj, None, orjson.OPT_NON_STR_KEYS | orjson.OPT_SORT_KEYS - ) + data_as_obj, + None, + orjson.OPT_NON_STR_KEYS | orjson.OPT_SORT_KEYS, + ), ) else: raise NotImplementedError @@ -140,7 +108,7 @@ for lib_name in LIBRARIES: f"{time_as_str:,.2f}" if time_as_str else "", f"{time_as_obj:,.2f}" if time_as_obj else "", f"{time_as_obj_sorted:,.2f}" if time_as_obj_sorted else "", - ) + ), ) buf = io.StringIO() diff --git a/script/pynumpy b/script/pynumpy index 77c0cd4f..4f7a9a7a 100755 --- a/script/pynumpy +++ b/script/pynumpy @@ -1,5 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2020-2025), Nazar Kostetskyi (2022), Aarni Koskela (2021) import gc import io @@ -9,53 +10,78 @@ import sys import time from timeit import timeit -import numpy +import numpy as np import psutil -import rapidjson -import simplejson -from memory_profiler import memory_usage from tabulate import tabulate import orjson -os.sched_setaffinity(os.getpid(), {0, 1}) +if hasattr(os, "sched_setaffinity"): + os.sched_setaffinity(os.getpid(), {0, 1}) kind = sys.argv[1] if len(sys.argv) >= 1 else "" -if kind == "int32": - array = numpy.random.randint(((2**31) - 1), size=(100000, 100), dtype=numpy.int32) + +if kind == "float16": + dtype = np.float16 + array = np.random.random(size=(50000, 100)).astype(dtype) +elif kind == "float32": + dtype = np.float32 + array = np.random.random(size=(50000, 100)).astype(dtype) elif kind == "float64": - array = numpy.random.random(size=(50000, 100)) - assert array.dtype == numpy.float64 + dtype = np.float64 + array = np.random.random(size=(50000, 100)) + assert array.dtype == np.float64 elif kind == "bool": - array = numpy.random.choice((True, False), size=(100000, 200)) + dtype = np.bool_ + array = np.random.choice((True, False), size=(100000, 200)) elif kind == "int8": - array = numpy.random.randint(((2**7) - 1), size=(100000, 100), dtype=numpy.int8) + dtype = np.int8 + array = np.random.randint(((2**7) - 1), size=(100000, 100), dtype=dtype) +elif kind == "int16": + dtype = np.int16 + array = np.random.randint(((2**15) - 1), size=(100000, 100), dtype=dtype) +elif kind == "int32": + dtype = np.int32 + array = np.random.randint(((2**31) - 1), size=(100000, 100), dtype=dtype) elif kind == "uint8": - array = numpy.random.randint(((2**8) - 1), size=(100000, 100), dtype=numpy.uint8) + dtype = np.uint8 + array = np.random.randint(((2**8) - 1), size=(100000, 100), dtype=dtype) +elif kind == "uint16": + dtype = np.uint16 + array = np.random.randint(((2**16) - 1), size=(100000, 100), dtype=dtype) +elif kind == "uint32": + dtype = np.uint32 + array = np.random.randint(((2**31) - 1), size=(100000, 100), dtype=dtype) else: - print("usage: pynumpy (bool|int32|float64|int8|uint8)") + print( + "usage: pynumpy (bool|int16|int32|float16|float32|float64|int8|uint8|uint16|uint32)", + ) sys.exit(1) proc = psutil.Process() def default(__obj): - if isinstance(__obj, numpy.ndarray): + if isinstance(__obj, np.ndarray): return __obj.tolist() + raise TypeError headers = ("Library", "Latency (ms)", "RSS diff (MiB)", "vs. orjson") -LIBRARIES = ("orjson", "ujson", "rapidjson", "simplejson", "json") +LIBRARIES = ("orjson", "json") ITERATIONS = 10 -orjson_dumps = lambda: orjson.dumps(array, option=orjson.OPT_SERIALIZE_NUMPY) -ujson_dumps = None -rapidjson_dumps = lambda: rapidjson.dumps(array, default=default).encode("utf-8") -simplejson_dumps = lambda: simplejson.dumps(array, default=default).encode("utf-8") -json_dumps = lambda: json.dumps(array, default=default).encode("utf-8") + +def orjson_dumps(): + return orjson.dumps(array, option=orjson.OPT_SERIALIZE_NUMPY) + + +def json_dumps(): + return json.dumps(array, default=default).encode("utf-8") + output_in_mib = len(orjson_dumps()) / 1024 / 1024 @@ -72,7 +98,7 @@ def per_iter_latency(val): def test_correctness(func): - return orjson.loads(func()) == array.tolist() + return np.array_equal(array, np.array(orjson.loads(func()), dtype=dtype)) table = [] @@ -93,7 +119,7 @@ for lib_name in LIBRARIES: ) latency = per_iter_latency(total_latency) time.sleep(1) - mem = max(memory_usage((func,), interval=0.001, timeout=latency * 2)) + mem = 0 # todo correct = test_correctness(func) if lib_name == "orjson": @@ -116,7 +142,7 @@ for lib_name in LIBRARIES: f"{latency:,.0f}" if latency else "", f"{mem_diff:,.0f}" if mem else "", f"{compared_to_orjson:,.1f}" if (latency and compared_to_orjson) else "", - ) + ), ) buf = io.StringIO() diff --git a/script/pysort b/script/pysort index b1a47204..ed8df4fb 100755 --- a/script/pysort +++ b/script/pysort @@ -1,5 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2019-2024), Aarni Koskela (2021) import io import json @@ -8,14 +9,12 @@ import os from pathlib import Path from timeit import timeit -import rapidjson -import simplejson -import ujson from tabulate import tabulate import orjson -os.sched_setaffinity(os.getpid(), {0, 1}) +if hasattr(os, "sched_setaffinity"): + os.sched_setaffinity(os.getpid(), {0, 1}) dirname = os.path.join(os.path.dirname(__file__), "..", "data") @@ -34,7 +33,7 @@ data = read_fixture_obj("twitter.json.xz") headers = ("Library", "unsorted (ms)", "sorted (ms)", "vs. orjson") -LIBRARIES = ("orjson", "ujson", "rapidjson", "simplejson", "json") +LIBRARIES = ("orjson", "json") ITERATIONS = 500 @@ -56,33 +55,6 @@ for lib_name in LIBRARIES: lambda: json.dumps(data, sort_keys=True).encode("utf-8"), number=ITERATIONS, ) - elif lib_name == "simplejson": - time_unsorted = timeit( - lambda: simplejson.dumps(data).encode("utf-8"), - number=ITERATIONS, - ) - time_sorted = timeit( - lambda: simplejson.dumps(data, sort_keys=True).encode("utf-8"), - number=ITERATIONS, - ) - elif lib_name == "ujson": - time_unsorted = timeit( - lambda: ujson.dumps(data).encode("utf-8"), - number=ITERATIONS, - ) - time_sorted = timeit( - lambda: ujson.dumps(data, sort_keys=True).encode("utf-8"), - number=ITERATIONS, - ) - elif lib_name == "rapidjson": - time_unsorted = timeit( - lambda: rapidjson.dumps(data).encode("utf-8"), - number=ITERATIONS, - ) - time_sorted = timeit( - lambda: rapidjson.dumps(data, sort_keys=True).encode("utf-8"), - number=ITERATIONS, - ) elif lib_name == "orjson": time_unsorted = timeit(lambda: orjson.dumps(data), number=ITERATIONS) time_sorted = timeit( @@ -109,7 +81,7 @@ for lib_name in LIBRARIES: f"{time_unsorted:,.2f}" if time_unsorted else "", f"{time_sorted:,.2f}" if time_sorted else "", f"{compared_to_orjson:,.1f}" if compared_to_orjson else "", - ) + ), ) buf = io.StringIO() diff --git a/script/pytest b/script/pytest index 421abe30..e8e7e51d 100755 --- a/script/pytest +++ b/script/pytest @@ -1,3 +1,3 @@ #!/bin/sh -e -pytest -s -rxX test +PYTHONMALLOC="debug" pytest -s test diff --git a/script/valgrind b/script/valgrind new file mode 100755 index 00000000..776f7991 --- /dev/null +++ b/script/valgrind @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -eou pipefail + +valgrind "$@" pytest -v --ignore=test/test_memory.py test diff --git a/setup.py b/setup.py deleted file mode 100755 index 4dd87bef..00000000 --- a/setup.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python3 - -from setuptools import setup - -setup( - name="orjson", - url="https://github.com/ijl/orjson", -) diff --git a/src/alloc.rs b/src/alloc.rs new file mode 100644 index 00000000..37042cc4 --- /dev/null +++ b/src/alloc.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2025) + +use crate::ffi::{PyMem_Free, PyMem_Malloc, PyMem_Realloc}; +use core::alloc::{GlobalAlloc, Layout}; +use core::ffi::c_void; + +struct PyMemAllocator {} + +#[global_allocator] +static ALLOCATOR: PyMemAllocator = PyMemAllocator {}; + +unsafe impl Sync for PyMemAllocator {} + +unsafe impl GlobalAlloc for PyMemAllocator { + #[inline] + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + unsafe { PyMem_Malloc(layout.size()).cast::() } + } + + #[inline] + unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { + unsafe { PyMem_Free(ptr.cast::()) } + } + + #[inline] + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + unsafe { + let len = layout.size(); + let ptr = PyMem_Malloc(len).cast::(); + core::ptr::write_bytes(ptr, 0, len); + ptr + } + } + + #[inline] + unsafe fn realloc(&self, ptr: *mut u8, _layout: Layout, new_size: usize) -> *mut u8 { + unsafe { PyMem_Realloc(ptr.cast::(), new_size).cast::() } + } +} diff --git a/src/deserialize/backend/ffi.rs b/src/deserialize/backend/ffi.rs new file mode 100644 index 00000000..73d737d0 --- /dev/null +++ b/src/deserialize/backend/ffi.rs @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2022-2026) + +#[repr(C)] +pub(crate) struct yyjson_alc { + pub malloc: ::core::option::Option< + unsafe extern "C" fn( + ctx: *mut ::core::ffi::c_void, + size: usize, + ) -> *mut ::core::ffi::c_void, + >, + pub realloc: ::core::option::Option< + unsafe extern "C" fn( + ctx: *mut ::core::ffi::c_void, + ptr: *mut ::core::ffi::c_void, + size: usize, + ) -> *mut ::core::ffi::c_void, + >, + pub free: ::core::option::Option< + unsafe extern "C" fn(ctx: *mut ::core::ffi::c_void, ptr: *mut ::core::ffi::c_void), + >, + pub ctx: *mut ::core::ffi::c_void, +} + +#[allow(non_camel_case_types)] +pub(crate) type yyjson_read_code = u32; +pub(crate) const YYJSON_READ_SUCCESS: yyjson_read_code = 0; + +#[repr(C)] +pub(crate) struct yyjson_read_err { + pub code: yyjson_read_code, + pub msg: *const ::core::ffi::c_char, + pub pos: usize, +} + +#[repr(C)] +pub(crate) union yyjson_val_uni { + pub u64_: u64, + pub i64_: i64, + pub f64_: f64, + pub str_: *const ::core::ffi::c_char, + pub ptr: *mut ::core::ffi::c_void, + pub ofs: usize, +} + +#[repr(C)] +pub(crate) struct yyjson_val { + pub tag: u64, + pub uni: yyjson_val_uni, +} + +#[repr(C)] +pub(crate) struct yyjson_doc { + pub root: *mut yyjson_val, + pub alc: yyjson_alc, + pub dat_read: usize, + pub val_read: usize, + pub str_pool: *mut ::core::ffi::c_char, +} + +unsafe extern "C" { + pub fn yyjson_read_opts( + dat: *mut ::core::ffi::c_char, + len: usize, + alc: *const yyjson_alc, + err: *mut yyjson_read_err, + ) -> *mut yyjson_doc; + + pub fn yyjson_alc_pool_init( + alc: *mut yyjson_alc, + buf: *mut ::core::ffi::c_void, + size: usize, + ) -> bool; +} diff --git a/src/deserialize/backend/mod.rs b/src/deserialize/backend/mod.rs new file mode 100644 index 00000000..5eb09a74 --- /dev/null +++ b/src/deserialize/backend/mod.rs @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2024-2025) + +mod ffi; +mod yyjson; + +pub(crate) use yyjson::deserialize; diff --git a/src/deserialize/backend/yyjson.rs b/src/deserialize/backend/yyjson.rs new file mode 100644 index 00000000..0956a7fd --- /dev/null +++ b/src/deserialize/backend/yyjson.rs @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright ijl (2022-2026), Anders Kaseorg (2023) + +use super::ffi::{ + YYJSON_READ_SUCCESS, yyjson_alc, yyjson_alc_pool_init, yyjson_doc, yyjson_read_err, + yyjson_read_opts, yyjson_val, +}; +use crate::deserialize::DeserializeError; +use crate::deserialize::pyobject::get_unicode_key; +use crate::ffi::{ + PyBoolRef, PyDictRef, PyFloatRef, PyIntRef, PyListRef, PyMem_Free, PyMem_Malloc, PyNoneRef, + PyStrRef, +}; +use core::ffi::c_char; +use core::ptr::{NonNull, null, null_mut}; +use std::borrow::Cow; + +const YYJSON_TAG_BIT: u8 = 8; + +const YYJSON_VAL_SIZE: usize = core::mem::size_of::(); + +const TAG_ARRAY: u8 = 0b00000110; +const TAG_DOUBLE: u8 = 0b00010100; +const TAG_FALSE: u8 = 0b00000011; +const TAG_INT64: u8 = 0b00001100; +const TAG_NULL: u8 = 0b00000010; +const TAG_OBJECT: u8 = 0b00000111; +const TAG_STRING: u8 = 0b00000101; +const TAG_TRUE: u8 = 0b00001011; +const TAG_UINT64: u8 = 0b00000100; + +macro_rules! is_yyjson_tag { + ($elem:expr, $tag:expr) => { + unsafe { (*$elem).tag as u8 == $tag } + }; +} + +fn yyjson_doc_get_root(doc: *mut yyjson_doc) -> *mut yyjson_val { + unsafe { (*doc).root } +} + +fn unsafe_yyjson_get_len(val: *mut yyjson_val) -> usize { + unsafe { ((*val).tag >> YYJSON_TAG_BIT) as usize } +} + +fn unsafe_yyjson_get_first(ctn: *mut yyjson_val) -> *mut yyjson_val { + unsafe { ctn.add(1) } +} + +const MINIMUM_BUFFER_CAPACITY: usize = 4096; + +fn buffer_capacity_to_allocate(len: usize) -> usize { + // The max memory size is (json_size / 2 * 16 * 1.5 + padding). + (((len / 2) * 24) + 256 + (MINIMUM_BUFFER_CAPACITY - 1)) & !(MINIMUM_BUFFER_CAPACITY - 1) +} + +fn unsafe_yyjson_is_ctn(val: *mut yyjson_val) -> bool { + unsafe { (*val).tag as u8 & 0b00000110 == 0b00000110 } +} + +#[allow(clippy::cast_ptr_alignment)] +fn unsafe_yyjson_get_next_container(val: *mut yyjson_val) -> *mut yyjson_val { + unsafe { (val.cast::().add((*val).uni.ofs)).cast::() } +} + +#[allow(clippy::cast_ptr_alignment)] +fn unsafe_yyjson_get_next_non_container(val: *mut yyjson_val) -> *mut yyjson_val { + unsafe { (val.cast::().add(YYJSON_VAL_SIZE)).cast::() } +} + +pub(crate) fn deserialize( + data: &'static str, +) -> Result, DeserializeError<'static>> { + assume!(!data.is_empty()); + let buffer_capacity = buffer_capacity_to_allocate(data.len()); + let buffer_ptr = unsafe { PyMem_Malloc(buffer_capacity) }; + if buffer_ptr.is_null() { + return Err(DeserializeError::from_yyjson( + Cow::Borrowed("Not enough memory to allocate buffer for parsing"), + 0, + data, + )); + } + let mut alloc = yyjson_alc { + malloc: None, + realloc: None, + free: None, + ctx: null_mut(), + }; + unsafe { + yyjson_alc_pool_init(&raw mut alloc, buffer_ptr, buffer_capacity); + } + + let mut err = yyjson_read_err { + code: YYJSON_READ_SUCCESS, + msg: null(), + pos: 0, + }; + + let doc = unsafe { + yyjson_read_opts( + data.as_ptr().cast::().cast_mut(), + data.len(), + &raw const alloc, + &raw mut err, + ) + }; + if doc.is_null() { + unsafe { + PyMem_Free(buffer_ptr); + } + let msg: Cow = unsafe { core::ffi::CStr::from_ptr(err.msg).to_string_lossy() }; + #[allow(clippy::cast_possible_wrap)] + let pos = err.pos as i64; + return Err(DeserializeError::from_yyjson(msg, pos, data)); + } + let val = yyjson_doc_get_root(doc); + let pyval = { + if !unsafe_yyjson_is_ctn(val) { + cold_path!(); + match ElementType::from_tag(val) { + ElementType::String => parse_yy_string(val), + ElementType::Uint64 => parse_yy_u64(val), + ElementType::Int64 => parse_yy_i64(val), + ElementType::Double => parse_yy_f64(val), + ElementType::Null => PyNoneRef::none().as_non_null_ptr(), + ElementType::True => PyBoolRef::pytrue().as_non_null_ptr(), + ElementType::False => PyBoolRef::pyfalse().as_non_null_ptr(), + ElementType::Array | ElementType::Object => unreachable_unchecked!(), + } + } else if is_yyjson_tag!(val, TAG_ARRAY) { + let pyval = PyListRef::with_capacity(unsafe_yyjson_get_len(val)); + if unsafe_yyjson_get_len(val) > 0 { + populate_yy_array(pyval.clone(), val); + } + pyval.as_non_null_ptr() + } else { + let pyval = PyDictRef::with_capacity(unsafe_yyjson_get_len(val)); + if unsafe_yyjson_get_len(val) > 0 { + populate_yy_object(pyval.clone(), val); + } + pyval.as_non_null_ptr() + } + }; + unsafe { + PyMem_Free(buffer_ptr); + } + Ok(pyval) +} + +enum ElementType { + String, + Uint64, + Int64, + Double, + Null, + True, + False, + Array, + Object, +} + +impl ElementType { + fn from_tag(elem: *mut yyjson_val) -> Self { + match unsafe { (*elem).tag as u8 } { + TAG_STRING => Self::String, + TAG_UINT64 => Self::Uint64, + TAG_INT64 => Self::Int64, + TAG_DOUBLE => Self::Double, + TAG_NULL => Self::Null, + TAG_TRUE => Self::True, + TAG_FALSE => Self::False, + TAG_ARRAY => Self::Array, + TAG_OBJECT => Self::Object, + _ => unreachable_unchecked!(), + } + } +} + +#[inline(always)] +fn parse_yy_string(elem: *mut yyjson_val) -> NonNull { + PyStrRef::from_str(str_from_slice!( + (*elem).uni.str_.cast::(), + unsafe_yyjson_get_len(elem) + )) + .as_non_null_ptr() +} + +#[inline(always)] +fn parse_yy_u64(elem: *mut yyjson_val) -> NonNull { + PyIntRef::from_u64(unsafe { (*elem).uni.u64_ }).as_non_null_ptr() +} + +#[inline(always)] +fn parse_yy_i64(elem: *mut yyjson_val) -> NonNull { + PyIntRef::from_i64(unsafe { (*elem).uni.i64_ }).as_non_null_ptr() +} + +#[inline(always)] +fn parse_yy_f64(elem: *mut yyjson_val) -> NonNull { + PyFloatRef::from_f64(unsafe { (*elem).uni.f64_ }).as_non_null_ptr() +} + +#[inline(never)] +fn populate_yy_array(mut list: PyListRef, elem: *mut yyjson_val) { + unsafe { + let len = unsafe_yyjson_get_len(elem); + assume!(len >= 1); + let mut next = unsafe_yyjson_get_first(elem); + + for idx in 0..len { + let val = next; + if unsafe_yyjson_is_ctn(val) { + cold_path!(); + next = unsafe_yyjson_get_next_container(val); + if is_yyjson_tag!(val, TAG_ARRAY) { + let pyval = PyListRef::with_capacity(unsafe_yyjson_get_len(val)); + list.set(idx, pyval.as_ptr()); + if unsafe_yyjson_get_len(val) > 0 { + populate_yy_array(pyval.clone(), val); + } + } else { + let pyval = PyDictRef::with_capacity(unsafe_yyjson_get_len(val)); + list.set(idx, pyval.as_ptr()); + if unsafe_yyjson_get_len(val) > 0 { + populate_yy_object(pyval.clone(), val); + } + } + } else { + next = unsafe_yyjson_get_next_non_container(val); + let pyval = match ElementType::from_tag(val) { + ElementType::String => parse_yy_string(val), + ElementType::Uint64 => parse_yy_u64(val), + ElementType::Int64 => parse_yy_i64(val), + ElementType::Double => parse_yy_f64(val), + ElementType::Null => PyNoneRef::none().as_non_null_ptr(), + ElementType::True => PyBoolRef::pytrue().as_non_null_ptr(), + ElementType::False => PyBoolRef::pyfalse().as_non_null_ptr(), + ElementType::Array | ElementType::Object => unreachable_unchecked!(), + }; + list.set(idx, pyval.as_ptr()); + } + } + } +} + +#[inline(never)] +fn populate_yy_object(mut dict: PyDictRef, elem: *mut yyjson_val) { + unsafe { + let len = unsafe_yyjson_get_len(elem); + assume!(len >= 1); + let mut next_key = unsafe_yyjson_get_first(elem); + let mut next_val = next_key.add(1); + for _ in 0..len { + let val = next_val; + let pykey = { + let key_str = str_from_slice!( + (*next_key).uni.str_.cast::(), + unsafe_yyjson_get_len(next_key) + ); + get_unicode_key(key_str) + }; + if unsafe_yyjson_is_ctn(val) { + cold_path!(); + next_key = unsafe_yyjson_get_next_container(val); + next_val = next_key.add(1); + if is_yyjson_tag!(val, TAG_ARRAY) { + let pyval = PyListRef::with_capacity(unsafe_yyjson_get_len(val)); + dict.set(pykey, pyval.as_ptr()); + if unsafe_yyjson_get_len(val) > 0 { + populate_yy_array(pyval, val); + } + } else { + let pyval = PyDictRef::with_capacity(unsafe_yyjson_get_len(val)); + dict.set(pykey, pyval.as_ptr()); + if unsafe_yyjson_get_len(val) > 0 { + populate_yy_object(pyval.clone(), val); + } + } + } else { + next_key = unsafe_yyjson_get_next_non_container(val); + next_val = next_key.add(1); + let pyval = match ElementType::from_tag(val) { + ElementType::String => parse_yy_string(val), + ElementType::Uint64 => parse_yy_u64(val), + ElementType::Int64 => parse_yy_i64(val), + ElementType::Double => parse_yy_f64(val), + ElementType::Null => PyNoneRef::none().as_non_null_ptr(), + ElementType::True => PyBoolRef::pytrue().as_non_null_ptr(), + ElementType::False => PyBoolRef::pyfalse().as_non_null_ptr(), + ElementType::Array | ElementType::Object => unreachable_unchecked!(), + }; + dict.set(pykey, pyval.as_ptr()); + } + } + } +} diff --git a/src/deserialize/cache.rs b/src/deserialize/cache.rs index 39a57c20..ce20b0c2 100644 --- a/src/deserialize/cache.rs +++ b/src/deserialize/cache.rs @@ -1,45 +1,40 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2019-2026) -use crate::typeref::*; -use ahash::CallHasher; -use associative_cache::replacement::RoundRobinReplacement; -use associative_cache::*; -use once_cell::unsync::OnceCell; -use std::os::raw::c_void; +use crate::ffi::{Py_DECREF, Py_INCREF, PyStrRef}; +use associative_cache::{AssociativeCache, Capacity2048, HashDirectMapped, RoundRobinReplacement}; +use core::cell::OnceCell; #[repr(transparent)] -pub struct CachedKey { - ptr: *mut c_void, +pub(crate) struct CachedKey { + ptr: PyStrRef, } unsafe impl Send for CachedKey {} unsafe impl Sync for CachedKey {} impl CachedKey { - pub fn new(ptr: *mut pyo3_ffi::PyObject) -> CachedKey { - CachedKey { - ptr: ptr as *mut c_void, - } + pub const fn new(ptr: PyStrRef) -> CachedKey { + CachedKey { ptr: ptr } } - - pub fn get(&mut self) -> *mut pyo3_ffi::PyObject { - let ptr = self.ptr as *mut pyo3_ffi::PyObject; - ffi!(Py_INCREF(ptr)); - ptr + pub fn get(&mut self) -> PyStrRef { + let ptr = self.ptr.as_ptr(); + unsafe { + Py_INCREF(ptr); + } + self.ptr } } impl Drop for CachedKey { fn drop(&mut self) { - ffi!(Py_DECREF(self.ptr as *mut pyo3_ffi::PyObject)); + unsafe { + Py_DECREF(self.ptr.as_ptr().cast::()); + } } } -pub type KeyMap = - AssociativeCache; - -pub static mut KEY_MAP: OnceCell = OnceCell::new(); +pub(crate) type KeyMap = + AssociativeCache; -pub fn cache_hash(key: &[u8]) -> u64 { - <[u8]>::get_hash(&key, unsafe { &*HASH_BUILDER }) -} +pub(crate) static mut KEY_MAP: OnceCell = OnceCell::new(); diff --git a/src/deserialize/deserializer.rs b/src/deserialize/deserializer.rs index ceb78de5..6b06fd05 100644 --- a/src/deserialize/deserializer.rs +++ b/src/deserialize/deserializer.rs @@ -1,34 +1,44 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2024-2026) -use crate::deserialize::utf8::{read_buf_to_str, read_input_to_buf}; -use crate::deserialize::DeserializeError; -use crate::typeref::*; -use std::ptr::NonNull; +use super::DeserializeError; +use super::input::Utf8Buffer; +use crate::ffi::{PyDictRef, PyListRef, PyStrRef}; +use core::ptr::NonNull; -pub fn deserialize( - ptr: *mut pyo3_ffi::PyObject, -) -> Result, DeserializeError<'static>> { - let buffer = read_input_to_buf(ptr)?; - if unlikely!(buffer.len() == 2) { - if buffer == b"[]" { - return Ok(nonnull!(ffi!(PyList_New(0)))); - } else if buffer == b"{}" { - return Ok(nonnull!(ffi!(PyDict_New()))); - } else if buffer == b"\"\"" { - ffi!(Py_INCREF(EMPTY_UNICODE)); - unsafe { return Ok(nonnull!(EMPTY_UNICODE)) } - } - } - - let buffer_str = read_buf_to_str(buffer)?; +#[repr(transparent)] +pub struct Deserializer { + buffer: Utf8Buffer, +} - #[cfg(feature = "yyjson")] - { - crate::deserialize::yyjson::deserialize_yyjson(buffer_str) +impl Deserializer { + #[inline] + pub fn from_pyobject( + ptr: *mut crate::ffi::PyObject, + ) -> Result> { + let buffer = Utf8Buffer::from_pyobject(ptr)?; + debug_assert!(!buffer.as_str().is_empty()); + Ok(Self { buffer: buffer }) } - #[cfg(not(feature = "yyjson"))] - { - crate::deserialize::json::deserialize_json(buffer_str) + #[inline] + pub fn deserialize(&self) -> Result, DeserializeError<'static>> { + if self.buffer.len() == 2 { + cold_path!(); + match self.buffer.as_bytes() { + b"[]" => return Ok(PyListRef::with_capacity(0).as_non_null_ptr()), + b"{}" => return Ok(PyDictRef::new().as_non_null_ptr()), + b"\"\"" => return Ok(PyStrRef::empty().as_non_null_ptr()), + _ => {} + } + } + crate::deserialize::backend::deserialize(self.buffer.as_str()) } } + +pub(crate) fn deserialize( + ptr: *mut crate::ffi::PyObject, +) -> Result, DeserializeError<'static>> { + let deserializer = Deserializer::from_pyobject(ptr)?; + deserializer.deserialize() +} diff --git a/src/deserialize/error.rs b/src/deserialize/error.rs index 9b39d603..12683d08 100644 --- a/src/deserialize/error.rs +++ b/src/deserialize/error.rs @@ -1,83 +1,46 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright ijl (2022-2026), Eric Jolibois (2021) use std::borrow::Cow; -#[derive(Debug, Clone)] -pub struct DeserializeError<'a> { +pub(crate) struct DeserializeError<'a> { pub message: Cow<'a, str>, - pub line: usize, // start at 1 - pub column: usize, // start at 1 - pub data: &'a str, + pub data: Option<&'a str>, pub pos: i64, } impl<'a> DeserializeError<'a> { #[cold] - pub fn new(message: Cow<'a, str>, line: usize, column: usize, data: &'a str) -> Self { + pub fn invalid(message: Cow<'a, str>) -> Self { DeserializeError { - message, - line, - column, - data, + message: message, + data: None, pos: 0, } } + #[cold] - #[cfg(feature = "yyjson")] pub fn from_yyjson(message: Cow<'a, str>, pos: i64, data: &'a str) -> Self { DeserializeError { - message, - line: 0, - column: 0, - data, - pos, + message: message, + data: Some(data), + pos: pos, } } /// Return position of the error in the deserialized data #[cold] - #[cfg_attr(feature = "unstable-simd", optimize(size))] + #[cfg_attr(feature = "optimize", optimize(size))] pub fn pos(&self) -> i64 { - if self.pos != 0 { - // yyjson - return bytecount::num_chars(&self.data.as_bytes()[0..self.pos as usize]) as i64; - } - if self.line == 0 { - return 1; + match self.data { + Some(as_str) => { + #[allow(clippy::cast_sign_loss)] + let pos = self.pos as usize; + #[allow(clippy::cast_possible_wrap)] + let res = as_str[0..pos].chars().count() as i64; // stmt_expr_attributes + res + } + None => 0, } - - let val = self.data - .split('\n') - // take only the relevant lines - .take(self.line) - .enumerate() - .map(|(idx, s)| { - if idx == self.line - 1 { - // Last line: only characters until the column of the error are relevant. - // Note: Rust uses bytes whereas Python uses chars: we hence cannot - // directly use the `column` field - if self.column == 0 { return 0; } - - // Find a column we can safely slice on - let mut column = self.column - 1; - while column > 0 && !s.is_char_boundary(column) { - column -= 1; - } - - let chars_count = s[..column].chars().count(); - if chars_count == s.chars().count() - 1 { - chars_count + 1 - } else { - chars_count - } - } else { - // Other lines - s.chars().count() - } - }) - .sum::() - // add missed '\n' characters - + (self.line - 1); - val as i64 } } diff --git a/src/deserialize/input.rs b/src/deserialize/input.rs new file mode 100644 index 00000000..faa6d30f --- /dev/null +++ b/src/deserialize/input.rs @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +use crate::deserialize::DeserializeError; +#[cfg(all(CPython, not(Py_GIL_DISABLED)))] +use crate::ffi::{PyByteArrayRef, PyMemoryViewRef}; +use crate::ffi::{PyBytesRef, PyStrRef}; +use crate::util::INVALID_STR; +use std::borrow::Cow; + +#[cfg(all(CPython, not(Py_GIL_DISABLED)))] +const INPUT_TYPE_MESSAGE: &str = "Input must be bytes, bytearray, memoryview, or str"; + +#[cfg(all(CPython, Py_GIL_DISABLED))] +const INPUT_TYPE_MESSAGE: &str = "Input must be bytes or str"; + +#[cfg(not(CPython))] +const INPUT_TYPE_MESSAGE: &str = "Input must be bytes, bytearray, or str"; + +#[cfg_attr(not(Py_GIL_DISABLED), repr(transparent))] +pub struct Utf8Buffer { + buffer: &'static str, +} + +impl Utf8Buffer { + #[cfg(all(CPython, not(Py_GIL_DISABLED)))] + fn buffer_from_ptr( + ptr: *mut crate::ffi::PyObject, + ) -> Result, DeserializeError<'static>> { + if let Ok(ob) = PyBytesRef::from_ptr(ptr) { + Ok(ob.as_str()) + } else if let Ok(ob) = PyStrRef::from_ptr(ptr) { + Ok(ob.as_str()) + } else if let Ok(ob) = PyByteArrayRef::from_ptr(ptr) { + Ok(ob.as_str()) + } else if let Ok(ob) = PyMemoryViewRef::from_ptr(ptr) { + Ok(ob.as_str()) + } else { + Err(DeserializeError::invalid(Cow::Borrowed(INPUT_TYPE_MESSAGE))) + } + } + + #[cfg(any(not(CPython), Py_GIL_DISABLED))] + fn buffer_from_ptr( + ptr: *mut crate::ffi::PyObject, + ) -> Result, DeserializeError<'static>> { + if let Ok(ob) = PyBytesRef::from_ptr(ptr) { + Ok(ob.as_str()) + } else if let Ok(ob) = PyStrRef::from_ptr(ptr) { + Ok(ob.as_str()) + } else { + Err(DeserializeError::invalid(Cow::Borrowed(INPUT_TYPE_MESSAGE))) + } + } + + pub fn from_pyobject( + ptr: *mut crate::ffi::PyObject, + ) -> Result> { + debug_assert!(!ptr.is_null()); + match Utf8Buffer::buffer_from_ptr(ptr) { + Ok(Some(as_str)) => { + if as_str.is_empty() { + cold_path!(); + Err(DeserializeError::invalid(Cow::Borrowed( + "Input is a zero-length, empty document", + ))) + } else { + Ok(Self { buffer: as_str }) + } + } + Ok(None) => { + cold_path!(); + Err(DeserializeError::invalid(Cow::Borrowed(INVALID_STR))) + } + Err(_) => Err(DeserializeError::invalid(Cow::Borrowed(INPUT_TYPE_MESSAGE))), + } + } + + pub fn as_str(&self) -> &'static str { + self.buffer + } + + pub fn as_bytes(&self) -> &'static [u8] { + self.buffer.as_bytes() + } + + pub fn len(&self) -> usize { + self.buffer.len() + } +} diff --git a/src/deserialize/json.rs b/src/deserialize/json.rs deleted file mode 100644 index 06967a35..00000000 --- a/src/deserialize/json.rs +++ /dev/null @@ -1,168 +0,0 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -use crate::deserialize::cache::*; -use crate::deserialize::pyobject::*; -use crate::deserialize::DeserializeError; -use crate::unicode::*; -use serde::de::{self, DeserializeSeed, Deserializer, MapAccess, SeqAccess, Visitor}; -use smallvec::SmallVec; -use std::borrow::Cow; -use std::fmt; -use std::ptr::NonNull; - -pub fn deserialize_json( - data: &'static str, -) -> Result, DeserializeError<'static>> { - let mut deserializer = serde_json::Deserializer::from_str(data); - let seed = JsonValue {}; - match seed.deserialize(&mut deserializer) { - Ok(obj) => { - deserializer.end().map_err(|e| { - DeserializeError::new(Cow::Owned(e.to_string()), e.line(), e.column(), data) - })?; - Ok(obj) - } - Err(e) => Err(DeserializeError::new( - Cow::Owned(e.to_string()), - e.line(), - e.column(), - data, - )), - } -} - -#[derive(Clone, Copy)] -struct JsonValue; - -impl<'de> DeserializeSeed<'de> for JsonValue { - type Value = NonNull; - - fn deserialize(self, deserializer: D) -> Result - where - D: Deserializer<'de>, - { - deserializer.deserialize_any(self) - } -} - -impl<'de> Visitor<'de> for JsonValue { - type Value = NonNull; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("JSON") - } - - fn visit_unit(self) -> Result { - Ok(parse_none()) - } - - fn visit_bool(self, value: bool) -> Result - where - E: de::Error, - { - Ok(parse_bool(value)) - } - - fn visit_i64(self, value: i64) -> Result - where - E: de::Error, - { - Ok(parse_i64(value)) - } - - fn visit_u64(self, value: u64) -> Result - where - E: de::Error, - { - Ok(parse_u64(value)) - } - - fn visit_f64(self, value: f64) -> Result - where - E: de::Error, - { - Ok(parse_f64(value)) - } - - fn visit_borrowed_str(self, value: &str) -> Result - where - E: de::Error, - { - Ok(nonnull!(unicode_from_str(value))) - } - - fn visit_str(self, value: &str) -> Result - where - E: de::Error, - { - Ok(nonnull!(unicode_from_str(value))) - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: SeqAccess<'de>, - { - match seq.next_element_seed(self) { - Ok(None) => Ok(nonnull!(ffi!(PyList_New(0)))), - Ok(Some(elem)) => { - let mut elements: SmallVec<[*mut pyo3_ffi::PyObject; 8]> = - SmallVec::with_capacity(8); - elements.push(elem.as_ptr()); - while let Some(elem) = seq.next_element_seed(self)? { - elements.push(elem.as_ptr()); - } - let ptr = ffi!(PyList_New(elements.len() as isize)); - for (i, &obj) in elements.iter().enumerate() { - ffi!(PyList_SET_ITEM(ptr, i as isize, obj)); - } - Ok(nonnull!(ptr)) - } - Err(err) => Result::Err(err), - } - } - - fn visit_map(self, mut map: A) -> Result - where - A: MapAccess<'de>, - { - let dict_ptr = ffi!(PyDict_New()); - while let Some(key) = map.next_key::>()? { - let value = map.next_value_seed(self)?; - let pykey: *mut pyo3_ffi::PyObject; - let pyhash: pyo3_ffi::Py_hash_t; - if unlikely!(key.len() > 64) { - pykey = unicode_from_str(&key); - pyhash = hash_str(pykey); - } else { - let hash = cache_hash(key.as_bytes()); - { - let map = unsafe { - KEY_MAP - .get_mut() - .unwrap_or_else(|| unsafe { std::hint::unreachable_unchecked() }) - }; - let entry = map.entry(&hash).or_insert_with( - || hash, - || { - let pyob = unicode_from_str(&key); - hash_str(pyob); - CachedKey::new(pyob) - }, - ); - pykey = entry.get(); - pyhash = unsafe { (*pykey.cast::()).hash } - } - } - let _ = ffi!(_PyDict_SetItem_KnownHash( - dict_ptr, - pykey, - value.as_ptr(), - pyhash - )); - // counter Py_INCREF in insertdict - ffi!(Py_DECREF(pykey)); - ffi!(Py_DECREF(value.as_ptr())); - } - Ok(nonnull!(dict_ptr)) - } -} diff --git a/src/deserialize/mod.rs b/src/deserialize/mod.rs index 21c682e4..6cf3e207 100644 --- a/src/deserialize/mod.rs +++ b/src/deserialize/mod.rs @@ -1,18 +1,15 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright ijl (2020-2026), Eric Jolibois (2021) +mod backend; +#[cfg(not(Py_GIL_DISABLED))] mod cache; mod deserializer; mod error; +mod input; mod pyobject; -mod utf8; -#[cfg(not(feature = "yyjson"))] -mod json; - -#[cfg(feature = "yyjson")] -mod yyjson; - -pub use cache::KeyMap; -pub use cache::KEY_MAP; -pub use deserializer::deserialize; -pub use error::DeserializeError; +#[cfg(not(Py_GIL_DISABLED))] +pub(crate) use cache::{KEY_MAP, KeyMap}; +pub(crate) use deserializer::deserialize; +pub(crate) use error::DeserializeError; diff --git a/src/deserialize/pyobject.rs b/src/deserialize/pyobject.rs index 469e5f53..1bd0e8f9 100644 --- a/src/deserialize/pyobject.rs +++ b/src/deserialize/pyobject.rs @@ -1,46 +1,43 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2022-2026) -use crate::typeref::*; -use std::ptr::NonNull; +use crate::ffi::PyStrRef; -#[allow(dead_code)] +#[cfg(all(CPython, not(Py_GIL_DISABLED)))] #[inline(always)] -pub fn parse_bool(val: bool) -> NonNull { - if val { - parse_true() +pub(crate) fn get_unicode_key(key_str: &str) -> PyStrRef { + if key_str.len() > 64 { + cold_path!(); + PyStrRef::from_str_with_hash(key_str) } else { - parse_false() + assume!(key_str.len() <= 64); + let hash = xxhash_rust::xxh3::xxh3_64(key_str.as_bytes()); + unsafe { + let entry = crate::deserialize::cache::KEY_MAP + .get_mut() + .unwrap_or_else(|| unreachable_unchecked!()) + .entry(&hash) + .or_insert_with( + || hash, + || { + crate::deserialize::cache::CachedKey::new(PyStrRef::from_str_with_hash( + key_str, + )) + }, + ); + entry.get() + } } } +#[cfg(all(CPython, Py_GIL_DISABLED))] #[inline(always)] -pub fn parse_true() -> NonNull { - ffi!(Py_INCREF(TRUE)); - nonnull!(TRUE) +pub(crate) fn get_unicode_key(key_str: &str) -> PyStrRef { + PyStrRef::from_str_with_hash(key_str) } +#[cfg(not(CPython))] #[inline(always)] -pub fn parse_false() -> NonNull { - ffi!(Py_INCREF(FALSE)); - nonnull!(FALSE) -} -#[inline(always)] -pub fn parse_i64(val: i64) -> NonNull { - nonnull!(ffi!(PyLong_FromLongLong(val))) -} - -#[inline(always)] -pub fn parse_u64(val: u64) -> NonNull { - nonnull!(ffi!(PyLong_FromUnsignedLongLong(val))) -} - -#[inline(always)] -pub fn parse_f64(val: f64) -> NonNull { - nonnull!(ffi!(PyFloat_FromDouble(val))) -} - -#[inline(always)] -pub fn parse_none() -> NonNull { - ffi!(Py_INCREF(NONE)); - nonnull!(NONE) +pub(crate) fn get_unicode_key(key_str: &str) -> PyStrRef { + PyStrRef::from_str(key_str) } diff --git a/src/deserialize/utf8.rs b/src/deserialize/utf8.rs deleted file mode 100644 index d08ffd6f..00000000 --- a/src/deserialize/utf8.rs +++ /dev/null @@ -1,82 +0,0 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) -use crate::deserialize::DeserializeError; -use crate::error::INVALID_STR; -use crate::ffi::*; -use crate::typeref::*; -use crate::unicode::*; -use std::borrow::Cow; -use std::os::raw::c_char; - -#[cfg(target_arch = "x86_64")] -fn is_valid_utf8(buf: &[u8]) -> bool { - if std::is_x86_feature_detected!("sse4.2") { - simdutf8::basic::from_utf8(buf).is_ok() - } else { - encoding_rs::Encoding::utf8_valid_up_to(buf) == buf.len() - } -} - -#[cfg(all(target_arch = "aarch64", feature = "unstable-simd"))] -fn is_valid_utf8(buf: &[u8]) -> bool { - simdutf8::basic::from_utf8(buf).is_ok() -} - -#[cfg(all(target_arch = "aarch64", not(feature = "unstable-simd")))] -fn is_valid_utf8(buf: &[u8]) -> bool { - std::str::from_utf8(buf).is_ok() -} - -#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] -fn is_valid_utf8(buf: &[u8]) -> bool { - std::str::from_utf8(buf).is_ok() -} - -pub fn read_input_to_buf( - ptr: *mut pyo3_ffi::PyObject, -) -> Result<&'static [u8], DeserializeError<'static>> { - let obj_type_ptr = ob_type!(ptr); - let buffer: *const u8; - let length: usize; - if is_type!(obj_type_ptr, BYTES_TYPE) { - buffer = unsafe { PyBytes_AS_STRING(ptr) as *const u8 }; - length = unsafe { PyBytes_GET_SIZE(ptr) as usize }; - } else if is_type!(obj_type_ptr, STR_TYPE) { - let uni = unicode_to_str(ptr); - if unlikely!(uni.is_none()) { - return Err(DeserializeError::new(Cow::Borrowed(INVALID_STR), 0, 0, "")); - } - let as_str = uni.unwrap(); - buffer = as_str.as_ptr(); - length = as_str.len(); - } else if is_type!(obj_type_ptr, MEMORYVIEW_TYPE) { - let membuf = unsafe { PyMemoryView_GET_BUFFER(ptr) }; - if unsafe { pyo3_ffi::PyBuffer_IsContiguous(membuf, b'C' as c_char) == 0 } { - return Err(DeserializeError::new( - Cow::Borrowed("Input type memoryview must be a C contiguous buffer"), - 0, - 0, - "", - )); - } - buffer = unsafe { (*membuf).buf as *const u8 }; - length = unsafe { (*membuf).len as usize }; - } else if is_type!(obj_type_ptr, BYTEARRAY_TYPE) { - buffer = ffi!(PyByteArray_AsString(ptr)) as *const u8; - length = ffi!(PyByteArray_Size(ptr)) as usize; - } else { - return Err(DeserializeError::new( - Cow::Borrowed("Input must be bytes, bytearray, memoryview, or str"), - 0, - 0, - "", - )); - } - Ok(unsafe { std::slice::from_raw_parts(buffer, length) }) -} - -pub fn read_buf_to_str(contents: &[u8]) -> Result<&str, DeserializeError> { - if !is_valid_utf8(contents) { - return Err(DeserializeError::new(Cow::Borrowed(INVALID_STR), 0, 0, "")); - } - Ok(unsafe { std::str::from_utf8_unchecked(contents) }) -} diff --git a/src/deserialize/yyjson.rs b/src/deserialize/yyjson.rs deleted file mode 100644 index d649e267..00000000 --- a/src/deserialize/yyjson.rs +++ /dev/null @@ -1,247 +0,0 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -use crate::deserialize::cache::*; -use crate::deserialize::pyobject::*; -use crate::deserialize::DeserializeError; -use crate::typeref::*; -use crate::unicode::*; -use crate::yyjson::*; -use std::borrow::Cow; -use std::os::raw::c_char; -use std::ptr::null; -use std::ptr::NonNull; - -const YYJSON_TYPE_MASK: u8 = 0x07; -const YYJSON_SUBTYPE_MASK: u8 = 0x18; -const YYJSON_TAG_BIT: u8 = 8; - -const YYJSON_VAL_SIZE: usize = std::mem::size_of::(); - -const TYPE_AND_SUBTYPE_MASK: u8 = YYJSON_TYPE_MASK | YYJSON_SUBTYPE_MASK; - -const TAG_ARRAY: u8 = 0b00000110; -const TAG_DOUBLE: u8 = 0b00010100; -const TAG_FALSE: u8 = 0b00000011; -const TAG_INT64: u8 = 0b00001100; -const TAG_NULL: u8 = 0b00000010; -const TAG_OBJECT: u8 = 0b00000111; -const TAG_STRING: u8 = 0b00000101; -const TAG_TRUE: u8 = 0b00001011; -const TAG_UINT64: u8 = 0b00000100; - -fn yyjson_doc_get_root(doc: *mut yyjson_doc) -> *mut yyjson_val { - unsafe { (*doc).root } -} - -fn unsafe_yyjson_get_len(val: *mut yyjson_val) -> usize { - unsafe { ((*val).tag >> YYJSON_TAG_BIT) as usize } -} - -fn yyjson_obj_iter_get_val(key: *mut yyjson_val) -> *mut yyjson_val { - unsafe { key.add(1) } -} - -fn unsafe_yyjson_get_first(ctn: *mut yyjson_val) -> *mut yyjson_val { - unsafe { ctn.add(1) } -} - -fn yyjson_read_max_memory_usage(len: usize) -> usize { - (12 * len) + 256 -} - -fn unsafe_yyjson_is_ctn(val: *mut yyjson_val) -> bool { - unsafe { ((*val).tag as u8) & 0b00000110 == 0b00000110 } -} - -fn unsafe_yyjson_get_next(val: *mut yyjson_val) -> *mut yyjson_val { - unsafe { - let ofs: usize; - if unlikely!(unsafe_yyjson_is_ctn(val)) { - ofs = (*val).uni.ofs; - } else { - ofs = YYJSON_VAL_SIZE; - } - ((val as *mut u8).add(ofs)) as *mut yyjson_val - } -} - -fn yyjson_arr_iter_next(iter: &mut yyjson_arr_iter) -> *mut yyjson_val { - unsafe { - let val = (*iter).cur; - (*iter).cur = unsafe_yyjson_get_next(val); - (*iter).idx += 1; - val - } -} - -fn yyjson_obj_iter_next(iter: &mut yyjson_obj_iter) -> *mut yyjson_val { - unsafe { - let key = (*iter).cur; - (*iter).cur = unsafe_yyjson_get_next(key.add(1)); - (*iter).idx += 1; - key - } -} - -pub fn deserialize_yyjson( - data: &'static str, -) -> Result, DeserializeError<'static>> { - unsafe { - let allocator: *mut yyjson_alc; - if yyjson_read_max_memory_usage(data.as_bytes().len()) < YYJSON_BUFFER_SIZE { - allocator = std::ptr::addr_of_mut!(*YYJSON_ALLOC); - } else { - allocator = std::ptr::null_mut(); - } - let mut err = yyjson_read_err { - code: YYJSON_READ_SUCCESS, - msg: null(), - pos: 0, - }; - let doc: *mut yyjson_doc = yyjson_read_opts( - data.as_bytes().as_ptr() as *mut c_char, - data.as_bytes().len(), - YYJSON_READ_NOFLAG, - allocator, - &mut err, - ); - if unlikely!(doc.is_null()) { - let msg: Cow = std::ffi::CStr::from_ptr(err.msg).to_string_lossy(); - Err(DeserializeError::from_yyjson(msg, err.pos as i64, data)) - } else { - let root = yyjson_doc_get_root(doc); - let ret = parse_node(root); - yyjson_doc_free(doc); - Ok(ret) - } - } -} - -enum ElementType { - String, - Uint64, - Int64, - Double, - Null, - True, - False, - Array, - Object, -} - -impl ElementType { - fn from_tag(elem: *mut yyjson_val) -> Self { - match unsafe { (*elem).tag as u8 & TYPE_AND_SUBTYPE_MASK } { - TAG_STRING => Self::String, - TAG_UINT64 => Self::Uint64, - TAG_INT64 => Self::Int64, - TAG_DOUBLE => Self::Double, - TAG_NULL => Self::Null, - TAG_TRUE => Self::True, - TAG_FALSE => Self::False, - TAG_ARRAY => Self::Array, - TAG_OBJECT => Self::Object, - _ => unreachable!(), - } - } -} - -fn parse_yy_string(elem: *mut yyjson_val) -> NonNull { - nonnull!(unicode_from_str(str_from_slice!( - (*elem).uni.str_ as *const u8, - unsafe_yyjson_get_len(elem) - ))) -} - -#[inline(never)] -fn parse_yy_array(elem: *mut yyjson_val) -> NonNull { - unsafe { - let len = unsafe_yyjson_get_len(elem); - let list = ffi!(PyList_New(len as isize)); - if len == 0 { - return nonnull!(list); - } - let mut iter: yyjson_arr_iter = yyjson_arr_iter { - idx: 0, - max: len, - cur: unsafe_yyjson_get_first(elem), - }; - for idx in 0..=len.saturating_sub(1) { - let val = yyjson_arr_iter_next(&mut iter); - let each = parse_node(val); - ffi!(PyList_SET_ITEM(list, idx as isize, each.as_ptr())); - } - nonnull!(list) - } -} - -#[inline(never)] -fn parse_yy_object(elem: *mut yyjson_val) -> NonNull { - unsafe { - let len = unsafe_yyjson_get_len(elem); - if len == 0 { - return nonnull!(ffi!(PyDict_New())); - } - let dict = ffi!(_PyDict_NewPresized(len as isize)); - let mut iter = yyjson_obj_iter { - idx: 0, - max: len, - cur: unsafe_yyjson_get_first(elem), - obj: elem, - }; - for _ in 0..=len.saturating_sub(1) { - let key = yyjson_obj_iter_next(&mut iter); - let val = yyjson_obj_iter_get_val(key); - let key_str = str_from_slice!((*key).uni.str_ as *const u8, unsafe_yyjson_get_len(key)); - let pyval = parse_node(val); - let pykey: *mut pyo3_ffi::PyObject; - let pyhash: pyo3_ffi::Py_hash_t; - if unlikely!(key_str.len() > 64) { - pykey = unicode_from_str(&key_str); - pyhash = hash_str(pykey); - } else { - let hash = cache_hash(key_str.as_bytes()); - { - let map = unsafe { - KEY_MAP - .get_mut() - .unwrap_or_else(|| unsafe { std::hint::unreachable_unchecked() }) - }; - let entry = map.entry(&hash).or_insert_with( - || hash, - || { - let pyob = unicode_from_str(&key_str); - hash_str(pyob); - CachedKey::new(pyob) - }, - ); - pykey = entry.get(); - pyhash = unsafe { (*pykey.cast::()).hash } - } - }; - let _ = ffi!(_PyDict_SetItem_KnownHash( - dict, - pykey, - pyval.as_ptr(), - pyhash - )); - ffi!(Py_DECREF(pykey)); - ffi!(Py_DECREF(pyval.as_ptr())); - } - nonnull!(dict) - } -} - -pub fn parse_node(elem: *mut yyjson_val) -> NonNull { - match ElementType::from_tag(elem) { - ElementType::String => parse_yy_string(elem), - ElementType::Uint64 => parse_u64(unsafe { (*elem).uni.u64_ }), - ElementType::Int64 => parse_i64(unsafe { (*elem).uni.i64_ }), - ElementType::Double => parse_f64(unsafe { (*elem).uni.f64_ }), - ElementType::Null => parse_none(), - ElementType::True => parse_true(), - ElementType::False => parse_false(), - ElementType::Array => parse_yy_array(elem), - ElementType::Object => parse_yy_object(elem), - } -} diff --git a/src/error.rs b/src/error.rs deleted file mode 100644 index 06c3fe4c..00000000 --- a/src/error.rs +++ /dev/null @@ -1,3 +0,0 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -pub const INVALID_STR: &str = "str is not valid UTF-8: surrogates not allowed"; diff --git a/src/exception.rs b/src/exception.rs new file mode 100644 index 00000000..1d2f0242 --- /dev/null +++ b/src/exception.rs @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright ijl (2020-2026), Jack Amadeo (2023) + +use core::ptr::null_mut; + +use crate::deserialize::DeserializeError; +use crate::ffi::{Py_DECREF, PyErr_SetObject, PyIntRef, PyObject, PyStrRef, PyTupleRef}; +use crate::typeref::{JsonDecodeError, JsonEncodeError}; + +#[cold] +#[inline(never)] +#[cfg_attr(feature = "optimize", optimize(size))] +pub(crate) fn raise_loads_exception(err: DeserializeError) -> *mut PyObject { + unsafe { + let err_pos = PyIntRef::from_i64(err.pos()); + let err_msg = PyStrRef::from_str(&err.message); + let doc = match err.data { + Some(as_str) => PyStrRef::from_str(as_str), + None => PyStrRef::empty(), + }; + let mut args = PyTupleRef::with_capacity(3); + args.set(0, err_msg.as_ptr()); + args.set(1, doc.as_ptr()); + args.set(2, err_pos.as_ptr()); + PyErr_SetObject(JsonDecodeError, args.as_ptr()); + Py_DECREF(args.as_ptr()); + } + null_mut() +} + +#[cold] +#[inline(never)] +#[cfg_attr(feature = "optimize", optimize(size))] +pub(crate) fn raise_dumps_exception_fixed(msg: &str) -> *mut PyObject { + unsafe { + let err_msg = PyStrRef::from_str(msg); + PyErr_SetObject(JsonEncodeError, err_msg.as_ptr()); + Py_DECREF(err_msg.as_ptr()); + } + null_mut() +} + +#[cold] +#[inline(never)] +#[cfg_attr(feature = "optimize", optimize(size))] +#[cfg(Py_3_12)] +pub(crate) fn raise_dumps_exception_dynamic(err: &str) -> *mut PyObject { + unsafe { + let cause_exc: *mut PyObject = crate::ffi::PyErr_GetRaisedException(); + + let err_msg = PyStrRef::from_str(err); + PyErr_SetObject(JsonEncodeError, err_msg.as_ptr()); + Py_DECREF(err_msg.as_ptr()); + + if !cause_exc.is_null() { + let exc: *mut PyObject = crate::ffi::PyErr_GetRaisedException(); + crate::ffi::PyException_SetCause(exc, cause_exc); + crate::ffi::PyErr_SetRaisedException(exc); + } + } + null_mut() +} + +#[cold] +#[inline(never)] +#[cfg_attr(feature = "optimize", optimize(size))] +#[cfg(not(Py_3_12))] +pub(crate) fn raise_dumps_exception_dynamic(err: &str) -> *mut PyObject { + unsafe { + let mut cause_tp: *mut PyObject = null_mut(); + let mut cause_val: *mut PyObject = null_mut(); + let mut cause_traceback: *mut PyObject = null_mut(); + crate::ffi::PyErr_Fetch(&mut cause_tp, &mut cause_val, &mut cause_traceback); + + let err_msg = PyStrRef::from_str(err); + PyErr_SetObject(JsonEncodeError, err_msg.as_ptr()); + Py_DECREF(err_msg.as_ptr()); + + let mut tp: *mut PyObject = null_mut(); + let mut val: *mut PyObject = null_mut(); + let mut traceback: *mut PyObject = null_mut(); + crate::ffi::PyErr_Fetch(&mut tp, &mut val, &mut traceback); + crate::ffi::PyErr_NormalizeException(&mut tp, &mut val, &mut traceback); + + if !cause_tp.is_null() { + crate::ffi::PyErr_NormalizeException( + &mut cause_tp, + &mut cause_val, + &mut cause_traceback, + ); + crate::ffi::PyException_SetCause(val, cause_val); + Py_DECREF(cause_tp); + } + if !cause_traceback.is_null() { + Py_DECREF(cause_traceback); + } + + crate::ffi::PyErr_Restore(tp, val, traceback); + } + null_mut() +} diff --git a/src/ffi/buffer.rs b/src/ffi/buffer.rs index c8b515da..ecce1b00 100644 --- a/src/ffi/buffer.rs +++ b/src/ffi/buffer.rs @@ -1,18 +1,19 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright ijl (2021-2026), Baul (2020) -use pyo3_ffi::*; -use std::os::raw::c_int; +use crate::ffi::{Py_buffer, Py_hash_t, Py_ssize_t, PyObject, PyVarObject}; +use core::ffi::c_int; #[repr(C)] -pub struct _PyManagedBufferObject { - pub ob_base: *mut pyo3_ffi::PyObject, +pub(crate) struct _PyManagedBufferObject { + pub ob_base: *mut PyObject, pub flags: c_int, pub exports: Py_ssize_t, pub master: *mut Py_buffer, } #[repr(C)] -pub struct PyMemoryViewObject { +pub(crate) struct PyMemoryViewObject { pub ob_base: PyVarObject, pub mbuf: *mut _PyManagedBufferObject, pub hash: Py_hash_t, @@ -25,6 +26,6 @@ pub struct PyMemoryViewObject { #[allow(non_snake_case)] #[inline(always)] -pub unsafe fn PyMemoryView_GET_BUFFER(op: *mut PyObject) -> *const Py_buffer { - &(*op.cast::()).view +pub(crate) unsafe fn PyMemoryView_GET_BUFFER(op: *mut PyObject) -> *const Py_buffer { + unsafe { &raw const (*op.cast::()).view } } diff --git a/src/ffi/bytes.rs b/src/ffi/bytes.rs index bf70433f..40c438e4 100644 --- a/src/ffi/bytes.rs +++ b/src/ffi/bytes.rs @@ -1,23 +1,27 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2022-2026) -use pyo3_ffi::*; -use std::os::raw::c_char; +use crate::ffi::{Py_ssize_t, PyObject}; +use core::ffi::c_char; -#[repr(C)] -pub struct PyBytesObject { - pub ob_base: PyVarObject, - pub ob_shash: Py_hash_t, - pub ob_sval: [c_char; 1], +pub(crate) use pyo3_ffi::PyBytesObject; + +#[cfg(CPython)] +#[allow(non_snake_case)] +#[inline(always)] +pub(crate) unsafe fn PyBytes_AS_STRING(op: *mut PyObject) -> *const c_char { + unsafe { (&raw const (*op.cast::()).ob_sval).cast::() } } +#[cfg(not(CPython))] #[allow(non_snake_case)] #[inline(always)] -pub unsafe fn PyBytes_AS_STRING(op: *mut PyObject) -> *const c_char { - &(*op.cast::()).ob_sval as *const c_char +pub(crate) unsafe fn PyBytes_AS_STRING(op: *mut PyObject) -> *const c_char { + unsafe { pyo3_ffi::PyBytes_AsString(op) } } #[allow(non_snake_case)] #[inline(always)] -pub unsafe fn PyBytes_GET_SIZE(op: *mut PyObject) -> Py_ssize_t { - (*op.cast::()).ob_size +pub(crate) unsafe fn PyBytes_GET_SIZE(op: *mut PyObject) -> Py_ssize_t { + unsafe { super::compat::Py_SIZE(op) } } diff --git a/src/ffi/compat.rs b/src/ffi/compat.rs new file mode 100644 index 00000000..6974d19a --- /dev/null +++ b/src/ffi/compat.rs @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2024-2026) + +#[cfg(Py_GIL_DISABLED)] +#[allow(non_upper_case_globals)] +pub(crate) const _Py_IMMORTAL_REFCNT_LOCAL: u32 = u32::MAX; + +#[cfg(all(Py_3_14, target_pointer_width = "32"))] +#[allow(non_upper_case_globals)] +pub(crate) const _Py_IMMORTAL_MINIMUM_REFCNT: pyo3_ffi::Py_ssize_t = + ((1 as core::ffi::c_long) << (30 as core::ffi::c_long)) as pyo3_ffi::Py_ssize_t; + +#[cfg(all(Py_3_12, not(Py_3_14)))] +#[allow(non_upper_case_globals)] +pub(crate) const _Py_IMMORTAL_REFCNT: pyo3_ffi::Py_ssize_t = { + if cfg!(target_pointer_width = "64") { + core::ffi::c_uint::MAX as pyo3_ffi::Py_ssize_t + } else { + // for 32-bit systems, use the lower 30 bits (see comment in CPython's object.h) + (core::ffi::c_uint::MAX >> 2) as pyo3_ffi::Py_ssize_t + } +}; +#[cfg(all(Py_3_12, not(Py_GIL_DISABLED)))] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn _Py_IsImmortal(op: *mut pyo3_ffi::PyObject) -> core::ffi::c_int { + unsafe { + #[cfg(all(target_pointer_width = "64", not(Py_GIL_DISABLED)))] + { + (((*op).ob_refcnt.ob_refcnt as pyo3_ffi::PY_INT32_T) < 0) as core::ffi::c_int + } + + #[cfg(all(target_pointer_width = "32", not(Py_GIL_DISABLED)))] + { + #[cfg(not(Py_3_14))] + { + ((*op).ob_refcnt.ob_refcnt == _Py_IMMORTAL_REFCNT) as core::ffi::c_int + } + + #[cfg(Py_3_14)] + { + ((*op).ob_refcnt.ob_refcnt >= _Py_IMMORTAL_MINIMUM_REFCNT) as core::ffi::c_int + } + } + + #[cfg(Py_GIL_DISABLED)] + { + ((*op) + .ob_ref_local + .load(core::sync::atomic::Ordering::Relaxed) + == _Py_IMMORTAL_REFCNT_LOCAL) as core::ffi::c_int + } + } +} + +#[cfg(CPython)] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn PyDict_New(len: isize) -> *mut pyo3_ffi::PyObject { + unsafe { + if len > 8 { + _PyDict_NewPresized(len) + } else { + pyo3_ffi::PyDict_New() + } + } +} + +#[cfg(not(CPython))] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn PyDict_New(_len: isize) -> *mut pyo3_ffi::PyObject { + unsafe { pyo3_ffi::PyDict_New() } +} + +#[cfg(all(CPython, Py_3_14))] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn Py_HashBuffer( + ptr: *const core::ffi::c_void, + len: pyo3_ffi::Py_ssize_t, +) -> pyo3_ffi::Py_hash_t { + unsafe { pyo3_ffi::Py_HashBuffer(ptr, len) } +} + +#[cfg(all(CPython, not(Py_3_14)))] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn Py_HashBuffer( + ptr: *const core::ffi::c_void, + len: pyo3_ffi::Py_ssize_t, +) -> pyo3_ffi::Py_hash_t { + unsafe { pyo3_ffi::_Py_HashBytes(ptr, len) } +} + +#[cfg(not(Py_3_13))] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn PyLong_AsByteArray( + v: *mut pyo3_ffi::PyLongObject, + bytes: *mut core::ffi::c_uchar, + n: pyo3_ffi::Py_ssize_t, + little_endian: core::ffi::c_int, + is_signed: core::ffi::c_int, +) -> core::ffi::c_int { + unsafe { _PyLong_AsByteArray(v, bytes, n, little_endian, is_signed) } +} + +#[cfg(Py_3_13)] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn PyLong_AsByteArray( + v: *mut pyo3_ffi::PyLongObject, + bytes: *mut core::ffi::c_uchar, + n: pyo3_ffi::Py_ssize_t, + little_endian: core::ffi::c_int, + is_signed: core::ffi::c_int, +) -> core::ffi::c_int { + unsafe { _PyLong_AsByteArray(v, bytes, n, little_endian, is_signed, 0) } +} + +#[cfg(CPython)] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn Py_SIZE(op: *mut pyo3_ffi::PyObject) -> pyo3_ffi::Py_ssize_t { + unsafe { (*op.cast::()).ob_size } +} + +#[cfg(not(CPython))] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn Py_SIZE(op: *mut pyo3_ffi::PyObject) -> pyo3_ffi::Py_ssize_t { + unsafe { pyo3_ffi::Py_SIZE(op) } +} + +#[allow(unused)] +#[cfg(any(CPython, PyPy))] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn Py_SET_SIZE(op: *mut pyo3_ffi::PyVarObject, size: pyo3_ffi::Py_ssize_t) { + unsafe { (*op).ob_size = size } +} + +#[allow(unused)] +#[cfg(GraalPy)] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn Py_SET_SIZE(op: *mut pyo3_ffi::PyVarObject, size: pyo3_ffi::Py_ssize_t) { + unsafe { (*op)._ob_size_graalpy = size } +} + +#[cfg(CPython)] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn PyTuple_GET_ITEM( + op: *mut pyo3_ffi::PyObject, + i: pyo3_ffi::Py_ssize_t, +) -> *mut pyo3_ffi::PyObject { + unsafe { + *(*op.cast::()) + .ob_item + .as_ptr() + .offset(i) + } +} + +#[cfg(not(CPython))] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn PyTuple_GET_ITEM( + op: *mut pyo3_ffi::PyObject, + i: pyo3_ffi::Py_ssize_t, +) -> *mut pyo3_ffi::PyObject { + unsafe { pyo3_ffi::PyTuple_GetItem(op, i) } +} + +#[cfg(CPython)] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn PyTuple_SET_ITEM( + op: *mut pyo3_ffi::PyObject, + i: pyo3_ffi::Py_ssize_t, + v: *mut pyo3_ffi::PyObject, +) { + unsafe { + *(*(op.cast::())) + .ob_item + .as_mut_ptr() + .offset(i) = v; + } +} + +#[cfg(not(CPython))] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn PyTuple_SET_ITEM( + op: *mut pyo3_ffi::PyObject, + i: pyo3_ffi::Py_ssize_t, + v: *mut pyo3_ffi::PyObject, +) { + unsafe { + pyo3_ffi::PyTuple_SetItem(op, i, v); + } +} + +#[cfg(not(Py_GIL_DISABLED))] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn PyUnstable_Unicode_GET_CACHED_HASH( + op: *mut pyo3_ffi::PyObject, +) -> pyo3_ffi::Py_hash_t { + unsafe { + let hash = (*op.cast::()).hash; + debug_assert!(hash != -1); + hash + } +} + +#[cfg(Py_GIL_DISABLED)] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn PyUnstable_Unicode_GET_CACHED_HASH( + op: *mut pyo3_ffi::PyObject, +) -> pyo3_ffi::Py_hash_t { + unsafe { + let hash = + core::sync::atomic::AtomicIsize::new((*op.cast::()).hash) + .load(core::sync::atomic::Ordering::Relaxed); + debug_assert!(hash != -1); + hash + } +} + +#[cfg(CPython)] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn PyObject_Type(o: *mut pyo3_ffi::PyObject) -> *mut pyo3_ffi::PyTypeObject { + unsafe { (*o).ob_type } +} + +#[cfg(not(CPython))] +#[inline(always)] +#[allow(non_snake_case)] +pub(crate) unsafe fn PyObject_Type(o: *mut pyo3_ffi::PyObject) -> *mut pyo3_ffi::PyTypeObject { + unsafe { pyo3_ffi::Py_TYPE(o) } +} + +#[cfg(all(not(Py_GIL_DISABLED), not(Py_LIMITED_ABI)))] +#[allow(non_snake_case)] +pub(crate) unsafe fn PyType_GetFlags(type_: *mut pyo3_ffi::PyTypeObject) -> core::ffi::c_ulong { + unsafe { (*type_).tp_flags } +} + +#[cfg(Py_GIL_DISABLED)] +#[allow(non_snake_case)] +pub(crate) unsafe fn PyType_GetFlags(type_: *mut pyo3_ffi::PyTypeObject) -> core::ffi::c_ulong { + unsafe { + (*type_) + .tp_flags + .load(core::sync::atomic::Ordering::Relaxed) + } +} + +#[cfg(Py_LIMITED_ABI)] +#[allow(non_snake_case)] +pub(crate) unsafe fn PyType_GetFlags(type_: *mut pyo3_ffi::PyTypeObject) -> core::ffi::c_ulong { + unsafe { pyo3_ffi::PyType_GetFlags(type_) } +} + +unsafe extern "C" { + #[cfg(CPython)] + pub fn _PyBytes_Resize( + pv: *mut *mut pyo3_ffi::PyObject, + newsize: pyo3_ffi::Py_ssize_t, + ) -> core::ffi::c_int; + + #[cfg(CPython)] + pub fn _PyDict_NewPresized(minused: pyo3_ffi::Py_ssize_t) -> *mut pyo3_ffi::PyObject; + + #[cfg(CPython)] + pub fn _PyDict_Contains_KnownHash( + op: *mut pyo3_ffi::PyObject, + key: *mut pyo3_ffi::PyObject, + hash: pyo3_ffi::Py_hash_t, + ) -> core::ffi::c_int; + + #[cfg(all(CPython, not(Py_3_13)))] + pub(crate) fn _PyDict_SetItem_KnownHash( + mp: *mut pyo3_ffi::PyObject, + name: *mut pyo3_ffi::PyObject, + value: *mut pyo3_ffi::PyObject, + hash: pyo3_ffi::Py_hash_t, + ) -> core::ffi::c_int; + + #[cfg(all(CPython, Py_3_13))] + pub(crate) fn _PyDict_SetItem_KnownHash_LockHeld( + mp: *mut pyo3_ffi::PyDictObject, + name: *mut pyo3_ffi::PyObject, + value: *mut pyo3_ffi::PyObject, + hash: pyo3_ffi::Py_hash_t, + ) -> core::ffi::c_int; + + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "_PyPyLong_AsByteArrayO")] + pub(crate) fn _PyLong_AsByteArray( + v: *mut pyo3_ffi::PyLongObject, + bytes: *mut core::ffi::c_uchar, + n: pyo3_ffi::Py_ssize_t, + little_endian: core::ffi::c_int, + is_signed: core::ffi::c_int, + with_exceptions: core::ffi::c_int, + ) -> core::ffi::c_int; + + #[cfg(not(Py_3_13))] + #[cfg_attr(PyPy, link_name = "_PyPyLong_AsByteArrayO")] + pub(crate) fn _PyLong_AsByteArray( + v: *mut pyo3_ffi::PyLongObject, + bytes: *mut core::ffi::c_uchar, + n: pyo3_ffi::Py_ssize_t, + little_endian: core::ffi::c_int, + is_signed: core::ffi::c_int, + ) -> core::ffi::c_int; +} diff --git a/src/ffi/dict.rs b/src/ffi/dict.rs deleted file mode 100644 index 690b62d7..00000000 --- a/src/ffi/dict.rs +++ /dev/null @@ -1,9 +0,0 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -use pyo3_ffi::{PyDictObject, PyObject, Py_ssize_t}; - -#[allow(non_snake_case)] -#[inline(always)] -pub unsafe fn PyDict_GET_SIZE(op: *mut PyObject) -> Py_ssize_t { - (*op.cast::()).ma_used -} diff --git a/src/ffi/fragment.rs b/src/ffi/fragment.rs new file mode 100644 index 00000000..b26eeece --- /dev/null +++ b/src/ffi/fragment.rs @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2020-2026) + +use crate::ffi::{ + Py_DECREF, Py_INCREF, Py_TPFLAGS_DEFAULT, Py_TPFLAGS_IMMUTABLETYPE, Py_tp_dealloc, Py_tp_new, + PyErr_SetObject, PyExc_TypeError, PyObject, PyTupleRef, PyType_FromSpec, PyType_Slot, + PyType_Spec, PyTypeObject, PyUnicode_FromStringAndSize, +}; +use core::ffi::{c_char, c_void}; +use core::ptr::null_mut; + +#[cfg(Py_GIL_DISABLED)] +use core::sync::atomic::{AtomicIsize, AtomicU32}; + +#[cfg(Py_GIL_DISABLED)] +macro_rules! pymutex_new { + () => { + unsafe { core::mem::zeroed() } + }; +} + +#[repr(C)] +pub(crate) struct Fragment { + #[cfg(Py_GIL_DISABLED)] + pub ob_tid: usize, + #[cfg(all(Py_GIL_DISABLED, Py_3_14))] + pub ob_flags: u16, + #[cfg(all(Py_GIL_DISABLED, not(Py_3_14)))] + pub _padding: u16, + #[cfg(Py_GIL_DISABLED)] + pub ob_mutex: pyo3_ffi::PyMutex, + #[cfg(Py_GIL_DISABLED)] + pub ob_gc_bits: u8, + #[cfg(Py_GIL_DISABLED)] + pub ob_ref_local: AtomicU32, + #[cfg(Py_GIL_DISABLED)] + pub ob_ref_shared: AtomicIsize, + #[cfg(not(Py_GIL_DISABLED))] + pub ob_refcnt: pyo3_ffi::Py_ssize_t, + #[cfg(PyPy)] + pub ob_pypy_link: pyo3_ffi::Py_ssize_t, + pub ob_type: *mut pyo3_ffi::PyTypeObject, + pub contents: *mut pyo3_ffi::PyObject, +} + +#[cold] +#[inline(never)] +#[cfg_attr(feature = "optimize", optimize(size))] +fn raise_args_exception() { + unsafe { + let msg = "orjson.Fragment() takes exactly 1 positional argument"; + let err_msg = + PyUnicode_FromStringAndSize(msg.as_ptr().cast::(), msg.len().cast_signed()); + PyErr_SetObject(PyExc_TypeError, err_msg); + Py_DECREF(err_msg); + }; +} + +#[unsafe(no_mangle)] +#[cold] +#[cfg_attr(feature = "optimize", optimize(size))] +pub(crate) unsafe extern "C" fn orjson_fragment_tp_new( + _subtype: *mut PyTypeObject, + args: *mut PyObject, + kwds: *mut PyObject, +) -> *mut PyObject { + unsafe { + let argsob = PyTupleRef::from_ptr_unchecked(args); + if argsob.len() != 1 || !kwds.is_null() { + raise_args_exception(); + null_mut() + } else { + let contents = argsob.get(0); + Py_INCREF(contents); + let obj = Box::new(Fragment { + #[cfg(Py_GIL_DISABLED)] + ob_tid: 0, + #[cfg(all(Py_GIL_DISABLED, Py_3_14))] + ob_flags: 0, + #[cfg(all(Py_GIL_DISABLED, not(Py_3_14)))] + _padding: 0, + #[cfg(Py_GIL_DISABLED)] + ob_mutex: pymutex_new!(), + #[cfg(Py_GIL_DISABLED)] + ob_gc_bits: 0, + #[cfg(Py_GIL_DISABLED)] + ob_ref_local: AtomicU32::new(0), + #[cfg(Py_GIL_DISABLED)] + ob_ref_shared: AtomicIsize::new(0), + #[cfg(not(Py_GIL_DISABLED))] + ob_refcnt: 1, + #[cfg(PyPy)] + ob_pypy_link: 0, + ob_type: crate::typeref::FRAGMENT_TYPE, + contents: contents, + }); + Box::into_raw(obj).cast::() + } + } +} + +#[unsafe(no_mangle)] +#[cold] +#[cfg_attr(feature = "optimize", optimize(size))] +pub(crate) unsafe extern "C" fn orjson_fragment_dealloc(object: *mut PyObject) { + unsafe { + Py_DECREF((*object.cast::()).contents); + crate::ffi::PyMem_Free(object.cast::()); + } +} + +#[unsafe(no_mangle)] +#[cold] +#[cfg_attr(feature = "optimize", optimize(size))] +pub(crate) unsafe extern "C" fn orjson_fragmenttype_new() -> *mut PyTypeObject { + unsafe { + let mut slots = [ + PyType_Slot { + slot: Py_tp_new, + pfunc: orjson_fragment_tp_new as *mut c_void, + }, + PyType_Slot { + slot: Py_tp_dealloc, + pfunc: orjson_fragment_dealloc as *mut c_void, + }, + PyType_Slot { + slot: 0, + pfunc: null_mut(), + }, + ]; + let mut spec = PyType_Spec { + name: c"orjson.Fragment".as_ptr(), + basicsize: core::mem::size_of::().cast_signed() as i32, + itemsize: 0, + flags: (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE) as u32, + slots: &raw mut slots[0], + }; + PyType_FromSpec(&raw mut spec).cast::() + } +} diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 2b7e2481..b33af1f1 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -1,11 +1,129 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2022-2026) +#[cfg(all(CPython, not(Py_GIL_DISABLED)))] mod buffer; mod bytes; -mod dict; -mod pytype; +pub(crate) mod compat; +mod fragment; +mod numpy; +mod pyboolref; +#[cfg(all(CPython, not(Py_GIL_DISABLED)))] +mod pybytearrayref; +mod pybytesref; +mod pydateref; +mod pydatetimeref; +mod pydictref; +mod pyfloatref; +mod pyfragmentref; +mod pyintref; +mod pylistref; +#[cfg(all(CPython, not(Py_GIL_DISABLED)))] +mod pymemoryview; +mod pynoneref; +mod pystrref; +mod pytimeref; +mod pytupleref; +mod pyuuidref; +mod utf8; -pub use buffer::*; -pub use bytes::*; -pub use dict::*; -pub use pytype::*; +pub(crate) use numpy::{ + NPY_ARRAY_C_CONTIGUOUS, NPY_ARRAY_NOTSWAPPED, NumpyBool, NumpyDateTimeError, NumpyDatetime64, + NumpyDatetime64Repr, NumpyDatetimeUnit, NumpyFloat16, NumpyFloat32, NumpyFloat64, NumpyInt8, + NumpyInt16, NumpyInt32, NumpyInt64, NumpyUint8, NumpyUint16, NumpyUint32, NumpyUint64, + PyArrayInterface, PyCapsule, +}; + +pub(crate) use compat::*; + +#[allow(unused_imports)] +pub(crate) use { + bytes::{PyBytes_AS_STRING, PyBytes_GET_SIZE, PyBytesObject}, + fragment::{Fragment, orjson_fragmenttype_new}, + pyboolref::PyBoolRef, + pybytesref::{PyBytesRef, PyBytesRefError}, + pydateref::PyDateRef, + pydatetimeref::PyDateTimeRef, + pydictref::PyDictRef, + pyfloatref::PyFloatRef, + pyfragmentref::{PyFragmentRef, PyFragmentRefError}, + pyintref::{PyIntError, PyIntKind, PyIntRef}, + pylistref::PyListRef, + pynoneref::PyNoneRef, + pystrref::{PyStrRef, PyStrSubclassRef, set_str_create_fn}, + pytimeref::PyTimeRef, + pytupleref::PyTupleRef, + pyuuidref::PyUuidRef, +}; + +#[allow(unused_imports)] +#[cfg(all(CPython, not(Py_GIL_DISABLED)))] +pub(crate) use { + pybytearrayref::{PyByteArrayRef, PyByteArrayRefError}, + pymemoryview::{PyMemoryViewRef, PyMemoryViewRefError}, +}; + +#[allow(unused_imports)] +pub(crate) use pyo3_ffi::{ + METH_FASTCALL, METH_KEYWORDS, METH_O, Py_DECREF, Py_False, Py_INCREF, Py_None, Py_REFCNT, + Py_TPFLAGS_DEFAULT, Py_TPFLAGS_DICT_SUBCLASS, Py_TPFLAGS_IMMUTABLETYPE, + Py_TPFLAGS_LIST_SUBCLASS, Py_TPFLAGS_LONG_SUBCLASS, Py_TPFLAGS_TUPLE_SUBCLASS, + Py_TPFLAGS_UNICODE_SUBCLASS, Py_TYPE, Py_True, Py_XDECREF, Py_buffer, Py_hash_t, Py_intptr_t, + Py_mod_exec, Py_ssize_t, Py_tp_dealloc, Py_tp_new, PyASCIIObject, PyBool_Type, + PyBuffer_IsContiguous, PyByteArray_AsString, PyByteArray_Size, PyByteArray_Type, + PyBytes_FromStringAndSize, PyBytes_Type, PyCFunction_NewEx, PyCapsule_Import, + PyCompactUnicodeObject, PyDateTime_CAPI, PyDateTime_DATE_GET_HOUR, + PyDateTime_DATE_GET_MICROSECOND, PyDateTime_DATE_GET_MINUTE, PyDateTime_DATE_GET_SECOND, + PyDateTime_DATE_GET_TZINFO, PyDateTime_DELTA_GET_DAYS, PyDateTime_DELTA_GET_SECONDS, + PyDateTime_DateTime, PyDateTime_GET_DAY, PyDateTime_GET_MONTH, PyDateTime_GET_YEAR, + PyDateTime_IMPORT, PyDateTime_TIME_GET_HOUR, PyDateTime_TIME_GET_MICROSECOND, + PyDateTime_TIME_GET_MINUTE, PyDateTime_TIME_GET_SECOND, PyDateTime_Time, PyDict_Contains, + PyDict_Next, PyDict_SetItem, PyDict_Type, PyDictObject, PyErr_Clear, PyErr_NewException, + PyErr_Occurred, PyErr_SetObject, PyExc_TypeError, PyException_SetCause, PyFloat_AS_DOUBLE, + PyFloat_FromDouble, PyFloat_Type, PyImport_ImportModule, PyList_GET_ITEM, PyList_New, + PyList_SET_ITEM, PyList_Type, PyListObject, PyLong_AsLong, PyLong_AsLongLong, + PyLong_AsUnsignedLongLong, PyLong_FromLongLong, PyLong_FromUnsignedLongLong, PyLong_Type, + PyLongObject, PyMapping_GetItemString, PyMem_Free, PyMem_Malloc, PyMem_Realloc, + PyMemoryView_Type, PyMethodDef, PyMethodDefPointer, PyModule_AddIntConstant, + PyModule_AddObject, PyModuleDef, PyModuleDef_HEAD_INIT, PyModuleDef_Init, PyModuleDef_Slot, + PyObject, PyObject_CallFunctionObjArgs, PyObject_CallMethodObjArgs, PyObject_GenericGetDict, + PyObject_GetAttr, PyObject_HasAttr, PyObject_Hash, PyObject_Vectorcall, PyTuple_New, + PyTuple_Type, PyTupleObject, PyType_FromSpec, PyType_Slot, PyType_Spec, PyTypeObject, + PyUnicode_AsUTF8AndSize, PyUnicode_FromStringAndSize, PyUnicode_InternFromString, + PyUnicode_New, PyUnicode_Type, PyVarObject, PyVectorcall_NARGS, +}; + +#[allow(unused_imports, deprecated)] +pub(crate) use pyo3_ffi::PyErr_Restore; + +#[cfg(CPython)] +pub(crate) use pyo3_ffi::{PyObject_CallMethodNoArgs, PyObject_CallMethodOneArg}; + +#[cfg(all(CPython, not(Py_GIL_DISABLED)))] +pub(crate) use buffer::PyMemoryView_GET_BUFFER; + +#[cfg(not(feature = "inline_str"))] +pub(crate) use pyo3_ffi::{PyUnicode_DATA, PyUnicode_KIND}; + +#[cfg(Py_3_12)] +#[allow(unused_imports)] +pub(crate) use pyo3_ffi::{ + Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED, Py_mod_multiple_interpreters, + PyErr_GetRaisedException, PyErr_SetRaisedException, PyType_GetDict, +}; + +#[cfg(not(Py_3_12))] +#[allow(unused_imports)] +pub(crate) use pyo3_ffi::{PyErr_Fetch, PyErr_NormalizeException}; + +#[cfg(not(Py_3_13))] +#[allow(unused_imports)] +pub(crate) use pyo3_ffi::PyModule_AddObjectRef; + +#[cfg(Py_3_13)] +#[allow(unused_imports)] +pub(crate) use pyo3_ffi::PyModule_Add; + +#[cfg(Py_3_13)] +#[allow(unused_imports)] +pub(crate) use pyo3_ffi::{Py_MOD_GIL_USED, Py_mod_gil}; diff --git a/src/ffi/numpy/array.rs b/src/ffi/numpy/array.rs new file mode 100644 index 00000000..9fa0c778 --- /dev/null +++ b/src/ffi/numpy/array.rs @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2020-2026) + +use crate::ffi::{Py_intptr_t, PyObject}; +use core::ffi::{c_char, c_int, c_void}; + +#[repr(C)] +pub(crate) struct PyCapsule { + head: PyObject, + pub pointer: *mut c_void, + pub name: *const c_char, + pub context: *mut c_void, + pub destructor: *mut c_void, // should be typedef void (*PyCapsule_Destructor)(PyObject *); +} + +// https://docs.scipy.org/doc/numpy/reference/arrays.interface.html#c.__array_struct__ + +pub(crate) const NPY_ARRAY_C_CONTIGUOUS: c_int = 0x1; +pub(crate) const NPY_ARRAY_NOTSWAPPED: c_int = 0x200; + +#[repr(C)] +pub(crate) struct PyArrayInterface { + pub two: c_int, + pub nd: c_int, + pub typekind: c_char, + pub itemsize: c_int, + pub flags: c_int, + pub shape: *mut Py_intptr_t, + pub strides: *mut Py_intptr_t, + pub data: *mut c_void, + pub descr: *mut PyObject, +} diff --git a/src/ffi/numpy/datetime.rs b/src/ffi/numpy/datetime.rs new file mode 100644 index 00000000..ab3104a3 --- /dev/null +++ b/src/ffi/numpy/datetime.rs @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright ijl (2022-2026), Ben Sully (2021) + +use crate::ffi::{Py_DECREF, PyListRef, PyObject, PyObject_GetAttr, PyStrRef, PyTupleRef}; +use crate::opt::Opt; +use crate::typeref::{DESCR_STR, DTYPE_STR}; +use jiff::Timestamp; +use jiff::civil::DateTime; + +/// This mimicks the units supported by numpy's datetime64 type. +/// +/// See +/// https://github.com/numpy/numpy/blob/fc8e3bbe419748ac5c6b7f3d0845e4bafa74644b/numpy/core/include/numpy/ndarraytypes.h#L268-L282. +#[derive(Clone, Copy, PartialEq)] +pub(crate) enum NumpyDatetimeUnit { + NaT, + Years, + Months, + Weeks, + Days, + Hours, + Minutes, + Seconds, + Milliseconds, + Microseconds, + Nanoseconds, + Picoseconds, + Femtoseconds, + Attoseconds, + Generic, +} + +impl NumpyDatetimeUnit { + #[cold] + pub const fn as_str(self) -> &'static str { + match self { + Self::NaT => "NaT", + Self::Years => "years", + Self::Months => "months", + Self::Weeks => "weeks", + Self::Days => "days", + Self::Hours => "hours", + Self::Minutes => "minutes", + Self::Seconds => "seconds", + Self::Milliseconds => "milliseconds", + Self::Microseconds => "microseconds", + Self::Nanoseconds => "nanoseconds", + Self::Picoseconds => "picoseconds", + Self::Femtoseconds => "femtoseconds", + Self::Attoseconds => "attoseconds", + Self::Generic => "generic", + } + } +} + +#[derive(Clone, Copy)] +pub(crate) enum NumpyDateTimeError { + UnsupportedUnit(NumpyDatetimeUnit), + Unrepresentable { unit: NumpyDatetimeUnit, val: i64 }, +} + +macro_rules! to_jiff_datetime { + ($timestamp:expr, $self:expr, $val:expr) => { + Ok( + ($timestamp.map_err(|_| NumpyDateTimeError::Unrepresentable { + unit: $self, + val: $val, + })?) + .to_zoned(jiff::tz::TimeZone::UTC) + .datetime(), + ) + }; +} + +impl NumpyDatetimeUnit { + /// Create a `NumpyDatetimeUnit` from a pointer to a Python object holding a + /// numpy array. + /// + /// This function must only be called with pointers to numpy arrays. + /// + /// We need to look inside the `obj.dtype.descr` attribute of the Python + /// object rather than using the `descr` field of the `__array_struct__` + /// because that field isn't populated for datetime64 arrays; see + /// https://github.com/numpy/numpy/issues/5350. + #[cold] + #[inline(never)] + pub fn from_pyobject(ptr: *mut PyObject) -> Self { + let dtype = unsafe { PyObject_GetAttr(ptr, DTYPE_STR) }; + let descr = unsafe { PyObject_GetAttr(dtype, DESCR_STR) }; + let el0 = unsafe { PyListRef::from_ptr_unchecked(descr).get(0) }; + let descr_str = unsafe { PyTupleRef::from_ptr_unchecked(el0).get(1) }; + match PyStrRef::from_ptr(descr_str) { + Ok(uni) => { + match uni.as_str() { + Some(as_str) => { + if as_str.len() < 5 { + return Self::NaT; + } + // unit descriptions are found at + // https://github.com/numpy/numpy/blob/b235f9e701e14ed6f6f6dcba885f7986a833743f/numpy/core/src/multiarray/datetime.c#L79-L96. + let ret = match &as_str[4..as_str.len() - 1] { + "Y" => Self::Years, + "M" => Self::Months, + "W" => Self::Weeks, + "D" => Self::Days, + "h" => Self::Hours, + "m" => Self::Minutes, + "s" => Self::Seconds, + "ms" => Self::Milliseconds, + "us" => Self::Microseconds, + "ns" => Self::Nanoseconds, + "ps" => Self::Picoseconds, + "fs" => Self::Femtoseconds, + "as" => Self::Attoseconds, + "generic" => Self::Generic, + _ => unreachable!(), + }; + unsafe { + Py_DECREF(dtype); + Py_DECREF(descr); + }; + ret + } + None => Self::NaT, + } + } + Err(_) => Self::NaT, + } + } + + #[cold] + #[cfg_attr(feature = "optimize", optimize(size))] + pub fn datetime(self, val: i64, opts: Opt) -> Result { + let datetime = match self { + Self::Years => { + let year = val + 1970; + if !(0..=9999).contains(&year) { + cold_path!(); + return Err(NumpyDateTimeError::Unrepresentable { unit: self, val }); + } else { + Ok(DateTime::new(year as i16, 1, 1, 0, 0, 0, 0).unwrap()) + } + } + Self::Months => { + let year = val / 12 + 1970; + let month = val % 12 + 1; + if !(0..=9999).contains(&year) || !(0..=12).contains(&month) { + cold_path!(); + return Err(NumpyDateTimeError::Unrepresentable { unit: self, val }); + } else { + Ok(DateTime::new(year as i16, month as i8, 1, 0, 0, 0, 0).unwrap()) + } + } + Self::Weeks => { + to_jiff_datetime!(Timestamp::from_second(val * 7 * 24 * 60 * 60), self, val) + } + Self::Days => to_jiff_datetime!(Timestamp::from_second(val * 24 * 60 * 60), self, val), + Self::Hours => to_jiff_datetime!(Timestamp::from_second(val * 60 * 60), self, val), + Self::Minutes => to_jiff_datetime!(Timestamp::from_second(val * 60), self, val), + Self::Seconds => to_jiff_datetime!(Timestamp::from_second(val), self, val), + Self::Milliseconds => to_jiff_datetime!(Timestamp::from_millisecond(val), self, val), + Self::Microseconds => to_jiff_datetime!(Timestamp::from_microsecond(val), self, val), + Self::Nanoseconds => { + to_jiff_datetime!(Timestamp::from_nanosecond(i128::from(val)), self, val) + } + _ => Err(NumpyDateTimeError::UnsupportedUnit(self)), + }; + match datetime { + Ok(dt) => match dt.year() { + 0..=9999 => Ok(NumpyDatetime64Repr { dt, opts }), + _ => Err(NumpyDateTimeError::Unrepresentable { unit: self, val }), + }, + Err(err) => Err(err), + } + } +} + +macro_rules! forward_inner { + ($meth: ident, $ty: ident) => { + pub fn $meth(&self) -> $ty { + debug_assert!(self.dt.$meth() >= 0); + #[allow(clippy::cast_sign_loss)] + let ret = self.dt.$meth() as $ty; // stmt_expr_attributes + ret + } + }; +} + +pub(crate) struct NumpyDatetime64Repr { + pub dt: DateTime, + pub opts: Opt, +} + +impl NumpyDatetime64Repr { + forward_inner!(year, i32); + forward_inner!(month, u8); + forward_inner!(day, u8); + forward_inner!(hour, u8); + forward_inner!(minute, u8); + forward_inner!(second, u8); + + pub fn nanosecond(&self) -> u32 { + debug_assert!(self.dt.subsec_nanosecond() >= 0); + self.dt.subsec_nanosecond().cast_unsigned() + } + + pub fn microsecond(&self) -> u32 { + self.nanosecond() / 1_000 + } +} diff --git a/src/ffi/numpy/mod.rs b/src/ffi/numpy/mod.rs new file mode 100644 index 00000000..2543486d --- /dev/null +++ b/src/ffi/numpy/mod.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +mod array; +mod datetime; +mod scalar; + +pub(crate) use array::{NPY_ARRAY_C_CONTIGUOUS, NPY_ARRAY_NOTSWAPPED, PyArrayInterface, PyCapsule}; +pub(crate) use datetime::{NumpyDateTimeError, NumpyDatetime64Repr, NumpyDatetimeUnit}; +pub(crate) use scalar::{ + NumpyBool, NumpyDatetime64, NumpyFloat16, NumpyFloat32, NumpyFloat64, NumpyInt8, NumpyInt16, + NumpyInt32, NumpyInt64, NumpyUint8, NumpyUint16, NumpyUint32, NumpyUint64, +}; diff --git a/src/ffi/numpy/scalar.rs b/src/ffi/numpy/scalar.rs new file mode 100644 index 00000000..f2dce921 --- /dev/null +++ b/src/ffi/numpy/scalar.rs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +use crate::ffi::PyObject; + +#[repr(C)] +pub(crate) struct NumpyFloat64 { + head: PyObject, + pub value: f64, +} + +#[repr(C)] +pub(crate) struct NumpyFloat32 { + head: PyObject, + pub value: f32, +} + +#[repr(C)] +pub(crate) struct NumpyFloat16 { + head: PyObject, + pub value: u16, +} + +#[repr(C)] +pub(crate) struct NumpyUint64 { + head: PyObject, + pub value: u64, +} + +#[repr(C)] +pub(crate) struct NumpyUint32 { + head: PyObject, + pub value: u32, +} + +#[repr(C)] +pub(crate) struct NumpyUint16 { + head: PyObject, + pub value: u16, +} + +#[repr(C)] +pub(crate) struct NumpyUint8 { + head: PyObject, + pub value: u8, +} + +#[repr(C)] +pub(crate) struct NumpyInt64 { + head: PyObject, + pub value: i64, +} + +#[repr(C)] +pub(crate) struct NumpyInt32 { + head: PyObject, + pub value: i32, +} + +#[repr(C)] +pub(crate) struct NumpyInt16 { + head: PyObject, + pub value: i16, +} + +#[repr(C)] +pub(crate) struct NumpyInt8 { + head: PyObject, + pub value: i8, +} + +#[repr(C)] +pub(crate) struct NumpyBool { + head: PyObject, + pub value: bool, +} + +#[repr(C)] +pub(crate) struct NumpyDatetime64 { + head: PyObject, + pub value: i64, +} diff --git a/src/ffi/pyboolref.rs b/src/ffi/pyboolref.rs new file mode 100644 index 00000000..bf780a19 --- /dev/null +++ b/src/ffi/pyboolref.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +#[derive(Clone)] +#[repr(transparent)] +pub(crate) struct PyBoolRef { + ptr: core::ptr::NonNull, +} + +unsafe impl Send for PyBoolRef {} +unsafe impl Sync for PyBoolRef {} + +impl PartialEq for PyBoolRef { + fn eq(&self, other: &Self) -> bool { + self.ptr == other.ptr + } +} + +impl PyBoolRef { + #[inline] + pub(crate) unsafe fn from_ptr_unchecked(ptr: *mut pyo3_ffi::PyObject) -> Self { + unsafe { + debug_assert!(!ptr.is_null()); + debug_assert!(crate::ffi::PyObject_Type(ptr) == crate::typeref::BOOL_TYPE); + Self { + ptr: core::ptr::NonNull::new_unchecked(ptr), + } + } + } + + #[inline] + #[allow(unused)] + pub const fn as_ptr(&self) -> *mut pyo3_ffi::PyObject { + self.ptr.as_ptr() + } + + #[inline] + #[allow(unused)] + pub const fn as_non_null_ptr(&self) -> core::ptr::NonNull { + self.ptr + } + + #[inline] + pub fn pytrue() -> Self { + Self { + ptr: unsafe { core::ptr::NonNull::new_unchecked(use_immortal!(crate::typeref::TRUE)) }, + } + } + + #[inline] + pub fn pyfalse() -> Self { + Self { + ptr: unsafe { core::ptr::NonNull::new_unchecked(use_immortal!(crate::typeref::FALSE)) }, + } + } +} diff --git a/src/ffi/pybytearrayref.rs b/src/ffi/pybytearrayref.rs new file mode 100644 index 00000000..b873ed39 --- /dev/null +++ b/src/ffi/pybytearrayref.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +pub(crate) enum PyByteArrayRefError { + NotType, +} + +#[derive(Clone)] +#[repr(transparent)] +pub(crate) struct PyByteArrayRef { + ptr: core::ptr::NonNull, +} + +unsafe impl Send for PyByteArrayRef {} +unsafe impl Sync for PyByteArrayRef {} + +impl PartialEq for PyByteArrayRef { + fn eq(&self, other: &Self) -> bool { + self.ptr == other.ptr + } +} + +impl PyByteArrayRef { + #[inline] + pub fn from_ptr(ptr: *mut pyo3_ffi::PyObject) -> Result { + unsafe { + debug_assert!(!ptr.is_null()); + if crate::ffi::PyObject_Type(ptr) == &raw mut crate::ffi::PyByteArray_Type { + Ok(Self { + ptr: core::ptr::NonNull::new_unchecked(ptr), + }) + } else { + Err(PyByteArrayRefError::NotType) + } + } + } + + #[inline] + pub fn as_ptr(&self) -> *mut pyo3_ffi::PyObject { + self.ptr.as_ptr() + } + + #[allow(unused)] + #[inline] + pub fn as_non_null_ptr(&self) -> core::ptr::NonNull { + self.ptr + } + + #[inline] + pub fn as_bytes(&self) -> &'static [u8] { + unsafe { + core::slice::from_raw_parts( + crate::ffi::PyByteArray_AsString(self.as_ptr()) + .cast::() + .cast_const(), + crate::util::isize_to_usize(crate::ffi::PyByteArray_Size(self.as_ptr())), + ) + } + } + + #[inline] + pub fn as_str(&self) -> Option<&'static str> { + let buffer = self.as_bytes(); + if !crate::ffi::utf8::is_valid_utf8(buffer) { + cold_path!(); + None + } else { + unsafe { Some(core::str::from_utf8_unchecked(buffer)) } + } + } +} diff --git a/src/ffi/pybytesref.rs b/src/ffi/pybytesref.rs new file mode 100644 index 00000000..d4f9e1d6 --- /dev/null +++ b/src/ffi/pybytesref.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +pub(crate) enum PyBytesRefError { + NotType, +} + +#[derive(Clone)] +#[repr(transparent)] +pub(crate) struct PyBytesRef { + ptr: core::ptr::NonNull, +} + +unsafe impl Send for PyBytesRef {} +unsafe impl Sync for PyBytesRef {} + +impl PartialEq for PyBytesRef { + fn eq(&self, other: &Self) -> bool { + self.ptr == other.ptr + } +} + +impl PyBytesRef { + #[inline] + pub fn from_ptr(ptr: *mut pyo3_ffi::PyObject) -> Result { + unsafe { + debug_assert!(!ptr.is_null()); + if crate::ffi::PyObject_Type(ptr) == crate::typeref::BYTES_TYPE { + Ok(Self { + ptr: core::ptr::NonNull::new_unchecked(ptr), + }) + } else { + Err(PyBytesRefError::NotType) + } + } + } + + #[inline] + pub fn as_ptr(&self) -> *mut pyo3_ffi::PyObject { + self.ptr.as_ptr() + } + + #[allow(unused)] + #[inline] + pub fn as_non_null_ptr(&self) -> core::ptr::NonNull { + self.ptr + } + + #[inline] + pub fn as_bytes(&self) -> &'static [u8] { + unsafe { + core::slice::from_raw_parts( + crate::ffi::PyBytes_AS_STRING(self.as_ptr()).cast::(), + crate::util::isize_to_usize(crate::ffi::PyBytes_GET_SIZE(self.as_ptr())), + ) + } + } + + #[inline] + pub fn as_str(&self) -> Option<&'static str> { + let buffer = self.as_bytes(); + if !crate::ffi::utf8::is_valid_utf8(buffer) { + cold_path!(); + None + } else { + unsafe { Some(core::str::from_utf8_unchecked(buffer)) } + } + } +} diff --git a/src/ffi/pydateref.rs b/src/ffi/pydateref.rs new file mode 100644 index 00000000..e591a40d --- /dev/null +++ b/src/ffi/pydateref.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +use crate::ffi::{PyDateTime_GET_DAY, PyDateTime_GET_MONTH, PyDateTime_GET_YEAR, PyObject}; + +#[derive(Clone)] +#[repr(transparent)] +pub(crate) struct PyDateRef { + ptr: core::ptr::NonNull, +} + +unsafe impl Send for PyDateRef {} +unsafe impl Sync for PyDateRef {} + +impl PartialEq for PyDateRef { + fn eq(&self, other: &Self) -> bool { + self.ptr == other.ptr + } +} + +impl PyDateRef { + #[inline] + pub(crate) unsafe fn from_ptr_unchecked(ptr: *mut PyObject) -> Self { + unsafe { + debug_assert!(!ptr.is_null()); + debug_assert!(crate::ffi::PyObject_Type(ptr) == crate::typeref::DATE_TYPE); + Self { + ptr: core::ptr::NonNull::new_unchecked(ptr), + } + } + } + + #[inline] + #[allow(unused)] + pub fn as_ptr(&self) -> *mut PyObject { + self.ptr.as_ptr() + } + + #[inline] + #[allow(unused)] + pub fn as_non_null_ptr(&self) -> core::ptr::NonNull { + self.ptr + } + + #[inline] + pub fn year(&self) -> u32 { + unsafe { + let tmp = PyDateTime_GET_YEAR(self.ptr.as_ptr()); + debug_assert!(tmp >= 0); + #[allow(clippy::cast_sign_loss)] + let val = tmp as u32; + val + } + } + + #[inline] + pub fn month(&self) -> u32 { + unsafe { + let tmp = PyDateTime_GET_MONTH(self.ptr.as_ptr()); + debug_assert!(tmp >= 0); + #[allow(clippy::cast_sign_loss)] + let val = tmp as u32; + val + } + } + + #[inline] + pub fn day(&self) -> u32 { + unsafe { + let tmp = PyDateTime_GET_DAY(self.ptr.as_ptr()); + debug_assert!(tmp >= 0); + #[allow(clippy::cast_sign_loss)] + let val = tmp as u32; + val + } + } +} diff --git a/src/ffi/pydatetimeref.rs b/src/ffi/pydatetimeref.rs new file mode 100644 index 00000000..26c0e07e --- /dev/null +++ b/src/ffi/pydatetimeref.rs @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright ijl (2025-2026), Ben Sully (2021) + +use crate::typeref::{ + CONVERT_METHOD_STR, DST_STR, NORMALIZE_METHOD_STR, UTCOFFSET_METHOD_STR, ZONEINFO_TYPE, +}; + +use crate::ffi::{PyObject_CallMethodNoArgs, PyObject_CallMethodOneArg, PyObject_HasAttr}; + +#[derive(Default)] +pub(crate) struct Offset { + pub day: i32, + pub second: i32, +} + +#[derive(Clone)] +#[repr(transparent)] +pub(crate) struct PyDateTimeRef { + ptr: core::ptr::NonNull, +} + +unsafe impl Send for PyDateTimeRef {} +unsafe impl Sync for PyDateTimeRef {} + +impl PartialEq for PyDateTimeRef { + fn eq(&self, other: &Self) -> bool { + self.ptr == other.ptr + } +} + +impl PyDateTimeRef { + #[inline] + pub(crate) unsafe fn from_ptr_unchecked(ptr: *mut pyo3_ffi::PyObject) -> Self { + unsafe { + debug_assert!(!ptr.is_null()); + debug_assert!(crate::ffi::PyObject_Type(ptr) == crate::typeref::DATETIME_TYPE); + Self { + ptr: core::ptr::NonNull::new_unchecked(ptr), + } + } + } + + #[inline] + #[allow(unused)] + pub fn as_ptr(&self) -> *mut pyo3_ffi::PyObject { + self.ptr.as_ptr() + } + + #[inline] + #[allow(unused)] + pub fn as_non_null_ptr(&self) -> core::ptr::NonNull { + self.ptr + } + + #[inline] + #[cfg(CPython)] + pub fn tzinfo(&self) -> *mut crate::ffi::PyObject { + unsafe { + let ret = (*(self.ptr.as_ptr().cast::())).tzinfo; + debug_assert!(!ret.is_null()); + ret + } + } + + #[inline] + #[cfg(not(CPython))] + pub fn tzinfo(&self) -> *mut crate::ffi::PyObject { + unsafe { + let ret = crate::ffi::PyDateTime_DATE_GET_TZINFO(self.ptr.as_ptr()); + debug_assert!(!ret.is_null()); + ret + } + } + + #[inline] + #[cfg(CPython)] + pub fn has_tz(&self) -> bool { + unsafe { (*(self.ptr.as_ptr().cast::())).hastzinfo == 1 } + } + + #[inline] + #[cfg(not(CPython))] + pub fn has_tz(&self) -> bool { + unsafe { self.tzinfo() != crate::typeref::NONE } + } + + #[inline] + pub fn year(&self) -> i32 { + unsafe { crate::ffi::PyDateTime_GET_YEAR(self.ptr.as_ptr()) } + } + + #[inline] + pub fn month(&self) -> u8 { + unsafe { + let tmp = crate::ffi::PyDateTime_GET_MONTH(self.ptr.as_ptr()); + debug_assert!(tmp >= 0); + #[allow(clippy::cast_sign_loss)] + let val = tmp as u8; + val + } + } + + #[inline] + pub fn day(&self) -> u8 { + unsafe { + let tmp = crate::ffi::PyDateTime_GET_DAY(self.ptr.as_ptr()); + debug_assert!(tmp >= 0); + #[allow(clippy::cast_sign_loss)] + let val = tmp as u8; + val + } + } + + #[inline] + pub fn hour(&self) -> u8 { + unsafe { + let tmp = crate::ffi::PyDateTime_DATE_GET_HOUR(self.ptr.as_ptr()); + debug_assert!(tmp >= 0); + #[allow(clippy::cast_sign_loss)] + let val = tmp as u8; + val + } + } + + #[inline] + pub fn minute(&self) -> u8 { + unsafe { + let tmp = crate::ffi::PyDateTime_DATE_GET_MINUTE(self.ptr.as_ptr()); + debug_assert!(tmp >= 0); + #[allow(clippy::cast_sign_loss)] + let val = tmp as u8; + val + } + } + + #[inline] + pub fn second(&self) -> u8 { + unsafe { + let tmp = crate::ffi::PyDateTime_DATE_GET_SECOND(self.ptr.as_ptr()); + debug_assert!(tmp >= 0); + #[allow(clippy::cast_sign_loss)] + let val = tmp as u8; + val + } + } + + #[inline] + pub fn microsecond(&self) -> u32 { + unsafe { + let tmp = crate::ffi::PyDateTime_DATE_GET_MICROSECOND(self.ptr.as_ptr()); + debug_assert!(tmp >= 0); + #[allow(clippy::cast_sign_loss)] + let val = tmp as u32; + val + } + } + #[cfg(not(CPython))] + #[inline] + pub fn offset(&self) -> Option { + unimplemented!() + } + + #[cfg(CPython)] + #[inline] + pub fn offset(&self) -> Option { + if !self.has_tz() { + Some(Offset::default()) + } else { + unsafe { + let tzinfo = self.tzinfo(); + if core::ptr::eq(crate::ffi::PyObject_Type(tzinfo), ZONEINFO_TYPE) { + // zoneinfo + let py_offset = + PyObject_CallMethodOneArg(tzinfo, UTCOFFSET_METHOD_STR, self.ptr.as_ptr()); + let offset = Offset { + second: crate::ffi::PyDateTime_DELTA_GET_SECONDS(py_offset), + day: crate::ffi::PyDateTime_DELTA_GET_DAYS(py_offset), + }; + crate::ffi::Py_DECREF(py_offset); + Some(offset) + } else { + self.slow_offset(tzinfo) + } + } + } + } + + #[cfg(CPython)] + #[cold] + #[inline(never)] + fn slow_offset(&self, tzinfo: *mut crate::ffi::PyObject) -> Option { + unsafe { + if PyObject_HasAttr(tzinfo, CONVERT_METHOD_STR) == 1 { + // pendulum + let py_offset = PyObject_CallMethodNoArgs(self.ptr.as_ptr(), UTCOFFSET_METHOD_STR); + let offset = Offset { + second: crate::ffi::PyDateTime_DELTA_GET_SECONDS(py_offset), + day: crate::ffi::PyDateTime_DELTA_GET_DAYS(py_offset), + }; + crate::ffi::Py_DECREF(py_offset); + Some(offset) + } else if PyObject_HasAttr(tzinfo, NORMALIZE_METHOD_STR) == 1 { + // pytz + let method_ptr = + PyObject_CallMethodOneArg(tzinfo, NORMALIZE_METHOD_STR, self.ptr.as_ptr()); + let py_offset = PyObject_CallMethodNoArgs(method_ptr, UTCOFFSET_METHOD_STR); + crate::ffi::Py_DECREF(method_ptr); + let offset = Offset { + second: crate::ffi::PyDateTime_DELTA_GET_SECONDS(py_offset), + day: crate::ffi::PyDateTime_DELTA_GET_DAYS(py_offset), + }; + crate::ffi::Py_DECREF(py_offset); + + Some(offset) + } else if PyObject_HasAttr(tzinfo, DST_STR) == 1 { + // dateutil/arrow, datetime.timezone.utc + let py_offset = + PyObject_CallMethodOneArg(tzinfo, UTCOFFSET_METHOD_STR, self.ptr.as_ptr()); + let offset = Offset { + second: crate::ffi::PyDateTime_DELTA_GET_SECONDS(py_offset), + day: crate::ffi::PyDateTime_DELTA_GET_DAYS(py_offset), + }; + crate::ffi::Py_DECREF(py_offset); + Some(offset) + } else { + None + } + } + } +} diff --git a/src/ffi/pydictref.rs b/src/ffi/pydictref.rs new file mode 100644 index 00000000..e9d42be1 --- /dev/null +++ b/src/ffi/pydictref.rs @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2025-2026) + +#[allow(unused)] +use super::{Py_TPFLAGS_DICT_SUBCLASS, PyType_GetFlags}; + +#[derive(Clone)] +#[repr(transparent)] +pub(crate) struct PyDictRef { + ptr: core::ptr::NonNull, +} + +unsafe impl Send for PyDictRef {} +unsafe impl Sync for PyDictRef {} + +impl PartialEq for PyDictRef { + fn eq(&self, other: &Self) -> bool { + self.ptr == other.ptr + } +} + +impl PyDictRef { + #[cfg(CPython)] + #[inline] + pub fn with_capacity(cap: usize) -> Self { + unsafe { + let ptr = crate::ffi::PyDict_New(crate::util::usize_to_isize(cap)); + debug_assert!(!ptr.is_null()); + Self { ptr: nonnull!(ptr) } + } + } + + #[cfg(not(CPython))] + #[allow(unused)] + #[inline] + pub fn with_capacity(_cap: usize) -> Self { + Self::new() + } + + #[allow(unused)] + #[inline] + pub fn new() -> Self { + unsafe { + let ptr = crate::ffi::PyDict_New(0); + debug_assert!(!ptr.is_null()); + Self { ptr: nonnull!(ptr) } + } + } + + #[inline] + pub(crate) unsafe fn from_ptr_unchecked(ptr: *mut pyo3_ffi::PyObject) -> Self { + unsafe { + debug_assert!(!ptr.is_null()); + debug_assert!( + crate::ffi::PyObject_Type(ptr) == crate::typeref::DICT_TYPE + || is_subclass_by_flag!( + PyType_GetFlags(crate::ffi::PyObject_Type(ptr)), + Py_TPFLAGS_DICT_SUBCLASS + ) + ); + Self { + ptr: core::ptr::NonNull::new_unchecked(ptr), + } + } + } + + #[inline] + #[allow(unused)] + pub fn as_ptr(&self) -> *mut pyo3_ffi::PyObject { + self.ptr.as_ptr() + } + + #[inline] + #[allow(unused)] + pub fn as_non_null_ptr(&self) -> core::ptr::NonNull { + self.ptr + } + + #[inline] + pub fn len(&self) -> usize { + unsafe { crate::util::isize_to_usize(super::Py_SIZE(self.as_ptr())) } + } + + #[cfg(CPython)] + #[inline] + pub fn set(&mut self, key: crate::ffi::PyStrRef, value: *mut crate::ffi::PyObject) { + debug_assert!(unsafe { crate::ffi::Py_REFCNT(self.as_ptr()) == 1 }); + debug_assert!(key.hash() != -1); + #[cfg(not(Py_3_13))] + unsafe { + let _ = crate::ffi::_PyDict_SetItem_KnownHash( + self.as_ptr(), + key.as_ptr(), + value, + key.hash(), + ); + } + #[cfg(Py_3_13)] + unsafe { + let _ = crate::ffi::_PyDict_SetItem_KnownHash_LockHeld( + self.as_ptr().cast::(), + key.as_ptr(), + value, + key.hash(), + ); + } + #[cfg(not(Py_GIL_DISABLED))] + reverse_pydict_incref!(key.as_ptr()); + reverse_pydict_incref!(value); + } + + #[cfg(not(CPython))] + #[inline] + pub fn set(&mut self, key: crate::ffi::PyStrRef, value: *mut crate::ffi::PyObject) { + unsafe { + let _ = crate::ffi::PyDict_SetItem(self.as_ptr(), key.as_ptr(), value); + } + #[cfg(not(Py_GIL_DISABLED))] + reverse_pydict_incref!(key.as_ptr()); + reverse_pydict_incref!(value); + } +} diff --git a/src/ffi/pyfloatref.rs b/src/ffi/pyfloatref.rs new file mode 100644 index 00000000..f1f80e83 --- /dev/null +++ b/src/ffi/pyfloatref.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +#[derive(Clone)] +#[repr(transparent)] +pub(crate) struct PyFloatRef { + ptr: core::ptr::NonNull, +} + +unsafe impl Send for PyFloatRef {} +unsafe impl Sync for PyFloatRef {} + +impl PartialEq for PyFloatRef { + fn eq(&self, other: &Self) -> bool { + self.ptr == other.ptr + } +} + +impl PyFloatRef { + #[inline] + pub(crate) unsafe fn from_ptr_unchecked(ptr: *mut pyo3_ffi::PyObject) -> Self { + unsafe { + debug_assert!(!ptr.is_null()); + debug_assert!(crate::ffi::PyObject_Type(ptr) == crate::typeref::FLOAT_TYPE); + Self { + ptr: core::ptr::NonNull::new_unchecked(ptr), + } + } + } + + #[inline] + pub fn as_non_null_ptr(&self) -> core::ptr::NonNull { + self.ptr + } + + #[inline] + pub fn value(&self) -> f64 { + unsafe { super::PyFloat_AS_DOUBLE(self.ptr.as_ptr()) } + } + + #[inline] + pub fn from_f64(value: f64) -> Self { + unsafe { + let ptr = super::PyFloat_FromDouble(value); + Self::from_ptr_unchecked(ptr) + } + } +} diff --git a/src/ffi/pyfragmentref.rs b/src/ffi/pyfragmentref.rs new file mode 100644 index 00000000..a6383885 --- /dev/null +++ b/src/ffi/pyfragmentref.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +use crate::ffi::{Fragment, PyBytesRef, PyStrRef}; + +pub(crate) enum PyFragmentRefError { + InvalidStr, + InvalidFragment, +} + +#[repr(transparent)] +pub(crate) struct PyFragmentRef { + ptr: core::ptr::NonNull, +} + +unsafe impl Send for PyFragmentRef {} +unsafe impl Sync for PyFragmentRef {} + +impl PartialEq for PyFragmentRef { + fn eq(&self, other: &Self) -> bool { + self.ptr == other.ptr + } +} + +impl PyFragmentRef { + #[inline] + pub(crate) unsafe fn from_ptr_unchecked(ptr: *mut pyo3_ffi::PyObject) -> Self { + unsafe { + debug_assert!(!ptr.is_null()); + debug_assert!(crate::ffi::PyObject_Type(ptr) == crate::typeref::FRAGMENT_TYPE); + Self { + ptr: core::ptr::NonNull::new_unchecked(ptr), + } + } + } + + #[inline] + #[allow(unused)] + pub fn as_ptr(&self) -> *mut pyo3_ffi::PyObject { + self.ptr.as_ptr() + } + + #[inline] + #[allow(unused)] + pub fn as_non_null_ptr(&self) -> core::ptr::NonNull { + self.ptr + } + + #[cold] + pub fn value(&self) -> Result<&[u8], PyFragmentRefError> { + let buffer: &[u8]; + unsafe { + let contents: *mut pyo3_ffi::PyObject = + (*self.ptr.as_ptr().cast::()).contents; + if let Ok(ob) = PyBytesRef::from_ptr(contents) { + buffer = ob.as_bytes(); + } else if let Ok(ob) = PyStrRef::from_ptr(contents) { + match ob.as_str() { + Some(ob) => buffer = ob.as_bytes(), + None => return Err(PyFragmentRefError::InvalidStr), + } + } else { + return Err(PyFragmentRefError::InvalidFragment); + } + Ok(buffer) + } + } +} diff --git a/src/ffi/pyintref.rs b/src/ffi/pyintref.rs new file mode 100644 index 00000000..ad8775a9 --- /dev/null +++ b/src/ffi/pyintref.rs @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2023-2026) + +#[allow(unused)] +use super::{Py_TPFLAGS_LONG_SUBCLASS, PyType_GetFlags}; +use super::{PyLong_FromLongLong, PyLong_FromUnsignedLongLong, PyObject}; +use crate::opt::{MAX_OPT, Opt}; + +// longintrepr.h, _longobject, _PyLongValue + +#[allow(dead_code)] +#[cfg(Py_3_12)] +#[allow(non_upper_case_globals)] +const SIGN_MASK: usize = 3; + +#[cfg(all(Py_3_12, feature = "inline_int"))] +#[allow(non_upper_case_globals)] +const NON_SIZE_BITS: usize = 3; + +#[cfg(Py_3_12)] +#[repr(C)] +pub(crate) struct _PyLongValue { + pub lv_tag: usize, + pub ob_digit: u32, +} + +#[cfg(all(Py_3_12, feature = "inline_int"))] +#[repr(C)] +pub(crate) struct PyLongObject { + pub ob_base: super::PyObject, + pub long_value: _PyLongValue, +} + +#[allow(dead_code)] +#[cfg(all(not(Py_3_12), feature = "inline_int"))] +#[repr(C)] +pub(crate) struct PyLongObject { + pub ob_base: super::PyVarObject, + pub ob_digit: u32, +} + +pub(crate) enum PyIntError { + NotType, + #[cfg(not(feature = "inline_int"))] + NotSigned, + Exceeds64Bit, +} + +pub(crate) enum PyIntOptConversionError { + InvalidRange, +} + +#[allow(unused)] +#[derive(PartialEq)] +pub(crate) enum PyIntKind { + U32, + I32, + U64, + I64, +} + +#[derive(Clone)] +#[repr(transparent)] +pub(crate) struct PyIntRef { + ptr: core::ptr::NonNull, +} + +unsafe impl Send for PyIntRef {} +unsafe impl Sync for PyIntRef {} + +impl PartialEq for PyIntRef { + fn eq(&self, other: &Self) -> bool { + self.ptr == other.ptr + } +} + +impl PyIntRef { + #[inline] + pub fn from_ptr(ptr: *mut pyo3_ffi::PyObject) -> Result { + unsafe { + debug_assert!(!ptr.is_null()); + if crate::ffi::PyObject_Type(ptr) == crate::typeref::INT_TYPE { + Ok(Self { + ptr: core::ptr::NonNull::new_unchecked(ptr), + }) + } else { + Err(PyIntError::NotType) + } + } + } + #[inline] + pub unsafe fn from_ptr_unchecked(ptr: *mut PyObject) -> Self { + unsafe { + debug_assert!(!ptr.is_null()); + debug_assert!( + crate::ffi::PyObject_Type(ptr) == crate::typeref::INT_TYPE + || is_subclass_by_flag!( + PyType_GetFlags(crate::ffi::PyObject_Type(ptr)), + Py_TPFLAGS_LONG_SUBCLASS + ) + ); + Self { + ptr: core::ptr::NonNull::new_unchecked(ptr), + } + } + } + + #[inline] + pub fn as_ptr(&self) -> *mut PyObject { + self.ptr.as_ptr() + } + + #[inline] + pub fn as_non_null_ptr(&self) -> core::ptr::NonNull { + self.ptr + } + + #[cfg(feature = "inline_int")] + #[inline] + pub fn kind(&self) -> PyIntKind { + match (self.is_signed(), self.fits_in_i32()) { + (true, true) => PyIntKind::I32, + (true, false) => PyIntKind::I64, + (false, true) => PyIntKind::U32, + (false, false) => PyIntKind::U64, + } + } + + #[cfg(all(Py_3_12, feature = "inline_int"))] + #[inline] + pub fn is_signed(&self) -> bool { + unsafe { (*self.as_ptr().cast::()).long_value.lv_tag & SIGN_MASK != 0 } + } + + #[cfg(all(not(Py_3_12), feature = "inline_int"))] + #[inline] + pub fn is_signed(&self) -> bool { + unsafe { (*self.as_ptr().cast::()).ob_size < 0 } + } + + #[cfg(all(Py_3_12, feature = "inline_int"))] + #[inline] + pub fn fits_in_i32(&self) -> bool { + unsafe { (*self.as_ptr().cast::()).long_value.lv_tag < (2 << NON_SIZE_BITS) } + } + + #[cfg(all(not(Py_3_12), feature = "inline_int"))] + #[inline] + pub fn fits_in_i32(&self) -> bool { + unsafe { isize::abs(super::Py_SIZE(self.as_ptr())) == 1 } + } + + #[cfg(all(Py_3_12, feature = "inline_int"))] + #[inline] + fn get_inline_value(&self) -> u32 { + unsafe { (*self.as_ptr().cast::()).long_value.ob_digit } + } + + #[cfg(all(not(Py_3_12), feature = "inline_int"))] + #[inline] + fn get_inline_value(&self) -> u32 { + unsafe { (*self.as_ptr().cast::()).ob_digit } + } + + #[cfg(feature = "inline_int")] + #[inline] + pub unsafe fn as_u32(&self) -> u32 { + debug_assert!(self.kind() == PyIntKind::U32); + self.get_inline_value() + } + + #[cfg(feature = "inline_int")] + #[inline] + pub unsafe fn as_i32(&self) -> i32 { + debug_assert!(self.kind() == PyIntKind::I32); + -(self.get_inline_value().cast_signed()) + } + + #[cfg(feature = "inline_int")] + #[inline] + fn get_64bit_value(&self) -> Result<[u8; 8], PyIntError> { + unsafe { + let mut buffer: [u8; 8] = [0; 8]; + let ret = crate::ffi::PyLong_AsByteArray( + self.as_ptr().cast::(), + buffer.as_mut_ptr().cast::(), + 8, + 1, + i32::from(self.is_signed()), + ); + if ret == -1 { + cold_path!(); + #[cfg(not(Py_3_13))] + unsafe { + crate::ffi::PyErr_Clear() + }; + Err(PyIntError::Exceeds64Bit) + } else { + Ok(buffer) + } + } + } + + #[cfg(feature = "inline_int")] + #[inline] + pub unsafe fn as_i64(&self) -> Result { + debug_assert!(self.kind() == PyIntKind::I64); + let val = self.get_64bit_value()?; + #[allow(unnecessary_transmutes)] + unsafe { + Ok(core::mem::transmute::<[u8; 8], i64>(val)) + } + } + + #[cfg(feature = "inline_int")] + #[inline] + pub unsafe fn as_u64(&self) -> Result { + debug_assert!(self.kind() == PyIntKind::U64); + let val = self.get_64bit_value()?; + #[allow(unnecessary_transmutes)] + unsafe { + Ok(core::mem::transmute::<[u8; 8], u64>(val)) + } + } + + #[cfg(not(feature = "inline_int"))] + #[inline] + pub unsafe fn as_i64(&self) -> Result { + let ival = unsafe { crate::ffi::PyLong_AsLongLong(self.as_ptr()) }; + if ival == -1 && unsafe { !crate::ffi::PyErr_Occurred().is_null() } { + cold_path!(); + unsafe { crate::ffi::PyErr_Clear() }; + Err(PyIntError::NotSigned) + } else { + Ok(ival) + } + } + + #[cfg(not(feature = "inline_int"))] + #[inline] + pub unsafe fn as_u64(&self) -> Result { + let uval = unsafe { crate::ffi::PyLong_AsUnsignedLongLong(self.as_ptr()) }; + if uval == u64::MAX && unsafe { !crate::ffi::PyErr_Occurred().is_null() } { + cold_path!(); + Err(PyIntError::Exceeds64Bit) + } else { + Ok(uval) + } + } + + #[cfg(feature = "inline_int")] + pub fn as_opt(&self) -> Result { + let val = self.get_inline_value(); + if val == 0 { + Ok(val as Opt) + } else { + match self.kind() { + PyIntKind::U32 => { + if !(0..=MAX_OPT as u32).contains(&val) { + Err(PyIntOptConversionError::InvalidRange) + } else { + Ok(val as Opt) + } + } + _ => Err(PyIntOptConversionError::InvalidRange), + } + } + } + + #[cfg(not(feature = "inline_int"))] + pub fn as_opt(&self) -> Result { + match unsafe { self.as_u64() } { + Ok(val) => { + if !(0..=MAX_OPT as u64).contains(&val) { + Err(PyIntOptConversionError::InvalidRange) + } else { + Ok(val as Opt) + } + } + Err(_) => Err(PyIntOptConversionError::InvalidRange), + } + } + + #[inline] + pub fn from_i64(value: i64) -> Self { + unsafe { + let ptr = PyLong_FromLongLong(value); + debug_assert!(!ptr.is_null()); + Self::from_ptr_unchecked(ptr) + } + } + + #[inline] + pub fn from_u64(value: u64) -> Self { + unsafe { + let ptr = PyLong_FromUnsignedLongLong(value); + debug_assert!(!ptr.is_null()); + Self::from_ptr_unchecked(ptr) + } + } +} diff --git a/src/ffi/pylistref.rs b/src/ffi/pylistref.rs new file mode 100644 index 00000000..0ad1719c --- /dev/null +++ b/src/ffi/pylistref.rs @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +#[derive(Clone)] +#[repr(transparent)] +pub(crate) struct PyListRef { + ptr: core::ptr::NonNull, +} + +unsafe impl Send for PyListRef {} +unsafe impl Sync for PyListRef {} + +impl PartialEq for PyListRef { + fn eq(&self, other: &Self) -> bool { + self.ptr == other.ptr + } +} + +impl PyListRef { + #[inline] + pub unsafe fn from_ptr_unchecked(ptr: *mut pyo3_ffi::PyObject) -> Self { + unsafe { + debug_assert!(!ptr.is_null()); + debug_assert!( + is_type!(crate::ffi::PyObject_Type(ptr), crate::typeref::LIST_TYPE) + || is_subclass_by_flag!( + crate::ffi::PyType_GetFlags(crate::ffi::PyObject_Type(ptr)), + Py_TPFLAGS_LIST_SUBCLASS + ) + ); + Self { + ptr: core::ptr::NonNull::new_unchecked(ptr), + } + } + } + + #[inline] + pub fn with_capacity(cap: usize) -> Self { + unsafe { + let list = super::PyList_New(crate::util::usize_to_isize(cap)); + debug_assert!(!list.is_null()); + Self { + ptr: nonnull!(list), + } + } + } + + #[inline] + #[allow(unused)] + pub fn as_ptr(&self) -> *mut pyo3_ffi::PyObject { + self.ptr.as_ptr() + } + + #[inline] + #[allow(unused)] + pub fn as_non_null_ptr(&self) -> core::ptr::NonNull { + self.ptr + } + + #[cfg(CPython)] + #[inline] + pub fn data_ptr(&self) -> *const *mut pyo3_ffi::PyObject { + unsafe { (*self.ptr.as_ptr().cast::()).ob_item } + } + + #[cfg(CPython)] + #[inline] + pub fn get(&mut self, i: usize) -> *mut pyo3_ffi::PyObject { + unsafe { *((self.data_ptr()).add(i)) } + } + + #[cfg(not(CPython))] + #[inline] + pub fn get(&mut self, i: usize) -> *mut pyo3_ffi::PyObject { + unsafe { pyo3_ffi::PyList_GetItem(self.ptr.as_ptr(), crate::util::usize_to_isize(i)) } + } + + #[cfg(CPython)] + #[inline] + pub fn set(&mut self, i: usize, val: *mut pyo3_ffi::PyObject) { + unsafe { + core::ptr::write(self.data_ptr().cast_mut().add(i), val); + } + } + + #[cfg(not(CPython))] + pub fn set(&mut self, i: usize, val: *mut pyo3_ffi::PyObject) { + unsafe { pyo3_ffi::PyList_SetItem(self.ptr.as_ptr(), crate::util::usize_to_isize(i), val) }; + } + + #[inline] + pub fn len(&self) -> usize { + unsafe { crate::util::isize_to_usize(super::Py_SIZE(self.ptr.as_ptr())) } + } +} diff --git a/src/ffi/pymemoryview.rs b/src/ffi/pymemoryview.rs new file mode 100644 index 00000000..90b463cd --- /dev/null +++ b/src/ffi/pymemoryview.rs @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2024-2026) + +#[allow(clippy::enum_variant_names)] +#[allow(unused)] +pub(crate) enum PyMemoryViewRefError { + NotType, + NotCContiguous, + NotSupported, +} + +#[derive(Clone)] +#[repr(transparent)] +pub(crate) struct PyMemoryViewRef { + ptr: core::ptr::NonNull, +} + +unsafe impl Send for PyMemoryViewRef {} +unsafe impl Sync for PyMemoryViewRef {} + +impl PartialEq for PyMemoryViewRef { + fn eq(&self, other: &Self) -> bool { + self.ptr == other.ptr + } +} + +impl PyMemoryViewRef { + #[cfg(CPython)] + #[inline] + pub fn from_ptr(ptr: *mut pyo3_ffi::PyObject) -> Result { + unsafe { + debug_assert!(!ptr.is_null()); + if crate::ffi::PyObject_Type(ptr) == &raw mut crate::ffi::PyMemoryView_Type { + let membuf = crate::ffi::PyMemoryView_GET_BUFFER(ptr); + #[allow(clippy::cast_possible_wrap)] + if crate::ffi::PyBuffer_IsContiguous(membuf, b'C' as core::ffi::c_char) == 0 { + return Err(PyMemoryViewRefError::NotCContiguous); + } + Ok(Self { + ptr: core::ptr::NonNull::new_unchecked(ptr), + }) + } else { + Err(PyMemoryViewRefError::NotType) + } + } + } + + #[cfg(not(CPython))] + #[inline] + pub fn from_ptr(ptr: *mut pyo3_ffi::PyObject) -> Result { + unsafe { + debug_assert!(!ptr.is_null()); + if crate::ffi::PyObject_Type(ptr) == &raw mut crate::ffi::PyMemoryView_Type { + Err(PyMemoryViewRefError::NotSupported) + } else { + Err(PyMemoryViewRefError::NotType) + } + } + } + + #[allow(unused)] + #[inline] + pub fn as_ptr(&self) -> *mut pyo3_ffi::PyObject { + self.ptr.as_ptr() + } + + #[allow(unused)] + #[inline] + pub fn as_non_null_ptr(&self) -> core::ptr::NonNull { + self.ptr + } + + #[cfg(CPython)] + #[inline] + pub fn as_bytes(&self) -> &'static [u8] { + unsafe { + let membuf = crate::ffi::PyMemoryView_GET_BUFFER(self.as_ptr()); + core::slice::from_raw_parts( + (*membuf).buf.cast::().cast_const(), + crate::util::isize_to_usize((*membuf).len), + ) + } + } + + #[cfg(CPython)] + #[inline] + pub fn as_str(&self) -> Option<&'static str> { + let buffer = self.as_bytes(); + if !crate::ffi::utf8::is_valid_utf8(buffer) { + cold_path!(); + None + } else { + unsafe { Some(core::str::from_utf8_unchecked(buffer)) } + } + } + + #[cfg(not(CPython))] + #[inline] + pub fn as_str(&self) -> Option<&'static str> { + None + } +} diff --git a/src/ffi/pynoneref.rs b/src/ffi/pynoneref.rs new file mode 100644 index 00000000..0815e7fd --- /dev/null +++ b/src/ffi/pynoneref.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +#[derive(Clone)] +#[repr(transparent)] +pub(crate) struct PyNoneRef { + ptr: core::ptr::NonNull, +} + +unsafe impl Send for PyNoneRef {} +unsafe impl Sync for PyNoneRef {} + +impl PartialEq for PyNoneRef { + fn eq(&self, other: &Self) -> bool { + self.ptr == other.ptr + } +} + +impl PyNoneRef { + #[inline] + #[allow(unused)] + pub(crate) unsafe fn from_ptr_unchecked(ptr: *mut pyo3_ffi::PyObject) -> Self { + unsafe { + debug_assert!(!ptr.is_null()); + debug_assert!(crate::ffi::PyObject_Type(ptr) == crate::typeref::NONE_TYPE); + Self { + ptr: core::ptr::NonNull::new_unchecked(ptr), + } + } + } + + #[inline] + #[allow(unused)] + pub fn as_ptr(&self) -> *mut pyo3_ffi::PyObject { + self.ptr.as_ptr() + } + + #[inline] + #[allow(unused)] + pub fn as_non_null_ptr(&self) -> core::ptr::NonNull { + self.ptr + } + + #[inline] + pub fn none() -> Self { + Self { + ptr: unsafe { core::ptr::NonNull::new_unchecked(use_immortal!(crate::typeref::NONE)) }, + } + } +} diff --git a/src/ffi/pystrref/avx512.rs b/src/ffi/pystrref/avx512.rs new file mode 100644 index 00000000..e4c76974 --- /dev/null +++ b/src/ffi/pystrref/avx512.rs @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2024-2026) + +use super::pyunicode_new::*; + +use core::arch::x86_64::{ + _mm512_and_si512, _mm512_cmpgt_epu8_mask, _mm512_cmpneq_epi8_mask, _mm512_loadu_epi8, + _mm512_mask_cmpneq_epi8_mask, _mm512_maskz_loadu_epi8, _mm512_max_epu8, _mm512_set1_epi8, +}; + +#[inline(never)] +#[target_feature(enable = "avx512f,avx512bw,avx512vl,bmi2")] +pub(crate) unsafe fn create_str_impl_avx512vl(buf: &str) -> *mut crate::ffi::PyObject { + unsafe { + const STRIDE: usize = 64; + + let buf_ptr = buf.as_bytes().as_ptr().cast::(); + let buf_len = buf.len(); + + assume!(buf_len > 0); + + let num_loops = buf_len / STRIDE; + let remainder = buf_len % STRIDE; + + let remainder_mask: u64 = !(u64::MAX << remainder); + let mut str_vec = _mm512_maskz_loadu_epi8(remainder_mask, buf_ptr); + let sptr = buf_ptr.add(remainder); + + for i in 0..num_loops { + str_vec = _mm512_max_epu8( + str_vec, + _mm512_loadu_epi8(sptr.add(STRIDE * i).cast::()), + ); + } + + #[allow(overflowing_literals)] + let vec_128 = _mm512_set1_epi8(0b10000000i8); + if _mm512_cmpgt_epu8_mask(str_vec, vec_128) == 0 { + pyunicode_ascii(buf.as_bytes().as_ptr(), buf_len) + } else { + #[allow(overflowing_literals)] + let is_four = _mm512_cmpgt_epu8_mask(str_vec, _mm512_set1_epi8(239i8)) != 0; + #[allow(overflowing_literals)] + let is_not_latin = _mm512_cmpgt_epu8_mask(str_vec, _mm512_set1_epi8(195i8)) != 0; + #[allow(overflowing_literals)] + let multibyte = _mm512_set1_epi8(0b11000000i8); + + let mut num_chars = _mm512_mask_cmpneq_epi8_mask( + remainder_mask, + _mm512_and_si512(_mm512_maskz_loadu_epi8(remainder_mask, buf_ptr), multibyte), + vec_128, + ) + .count_ones() as usize; + + for i in 0..num_loops { + num_chars += _mm512_cmpneq_epi8_mask( + _mm512_and_si512( + _mm512_loadu_epi8(sptr.add(STRIDE * i).cast::()), + multibyte, + ), + vec_128, + ) + .count_ones() as usize; + } + + if is_four { + pyunicode_fourbyte(buf, num_chars) + } else if is_not_latin { + pyunicode_twobyte(buf, num_chars) + } else { + pyunicode_onebyte(buf, num_chars) + } + } + } +} diff --git a/src/ffi/pystrref/mod.rs b/src/ffi/pystrref/mod.rs new file mode 100644 index 00000000..1f6f3674 --- /dev/null +++ b/src/ffi/pystrref/mod.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +#[cfg(feature = "avx512")] +#[cfg(CPython)] +mod avx512; +mod object; +#[cfg(CPython)] +mod pyunicode_new; +#[cfg(CPython)] +mod scalar; + +pub(crate) use object::{PyStrRef, PyStrSubclassRef, set_str_create_fn}; diff --git a/src/ffi/pystrref/object.rs b/src/ffi/pystrref/object.rs new file mode 100644 index 00000000..69f8d4da --- /dev/null +++ b/src/ffi/pystrref/object.rs @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2025-2026) + +#[allow(unused)] +use crate::ffi::{ + Py_HashBuffer, Py_SIZE, Py_ssize_t, PyASCIIObject, PyCompactUnicodeObject, PyObject, + PyUnicode_AsUTF8AndSize, +}; +#[cfg(all(CPython, not(feature = "inline_str")))] +use crate::ffi::{PyUnicode_DATA, PyUnicode_KIND}; +use crate::typeref::{EMPTY_UNICODE, STR_TYPE}; +use core::ptr::NonNull; + +fn to_str_via_ffi(op: *mut PyObject) -> Option<&'static str> { + let mut str_size: Py_ssize_t = 0; + let ptr = unsafe { PyUnicode_AsUTF8AndSize(op, &raw mut str_size).cast::() }; + if ptr.is_null() { + cold_path!(); + None + } else { + #[allow(clippy::cast_sign_loss)] + let str_usize = str_size as usize; + Some(str_from_slice!(ptr, str_usize)) + } +} + +#[cfg(all(CPython, feature = "avx512"))] +pub type StrDeserializer = unsafe fn(&str) -> *mut PyObject; + +#[cfg(all(CPython, feature = "avx512"))] +static mut STR_CREATE_FN: StrDeserializer = super::scalar::str_impl_kind_scalar; + +pub fn set_str_create_fn() { + unsafe { + #[cfg(all(CPython, feature = "avx512"))] + if std::is_x86_feature_detected!("avx512vl") { + STR_CREATE_FN = super::avx512::create_str_impl_avx512vl; + } + } +} + +#[cfg(all(feature = "inline_str", Py_3_14, Py_GIL_DISABLED))] +const STATE_KIND_SHIFT: usize = 8; + +#[cfg(all(feature = "inline_str", not(all(Py_3_14, Py_GIL_DISABLED))))] +const STATE_KIND_SHIFT: usize = 2; + +#[cfg(feature = "inline_str")] +const STATE_KIND_MASK: u32 = 7 << STATE_KIND_SHIFT; + +#[cfg(feature = "inline_str")] +const STATE_COMPACT_ASCII: u32 = + 1 << STATE_KIND_SHIFT | 1 << (STATE_KIND_SHIFT + 3) | 1 << (STATE_KIND_SHIFT + 4); + +pub(crate) enum PyStrRefError { + NotStrType, +} + +#[derive(Copy, Clone)] +#[repr(transparent)] +pub(crate) struct PyStrRef { + ptr: core::ptr::NonNull, +} + +unsafe impl Send for PyStrRef {} +unsafe impl Sync for PyStrRef {} + +impl PartialEq for PyStrRef { + fn eq(&self, other: &Self) -> bool { + self.ptr == other.ptr + } +} + +impl PyStrRef { + #[inline] + pub fn from_ptr(ptr: *mut pyo3_ffi::PyObject) -> Result { + unsafe { + debug_assert!(!ptr.is_null()); + if crate::ffi::PyObject_Type(ptr) == crate::typeref::STR_TYPE { + Ok(Self { + ptr: core::ptr::NonNull::new_unchecked(ptr), + }) + } else { + cold_path!(); + Err(PyStrRefError::NotStrType) + } + } + } + + #[inline] + pub unsafe fn from_ptr_unchecked(ptr: *mut pyo3_ffi::PyObject) -> Self { + unsafe { + debug_assert!(!ptr.is_null()); + debug_assert!(crate::ffi::PyObject_Type(ptr) == crate::typeref::STR_TYPE); + Self { + ptr: core::ptr::NonNull::new_unchecked(ptr), + } + } + } + + #[inline] + pub fn empty() -> Self { + unsafe { + Self { + ptr: nonnull!(use_immortal!(EMPTY_UNICODE)), + } + } + } + + #[inline] + pub fn as_ptr(&self) -> *mut pyo3_ffi::PyObject { + self.ptr.as_ptr() + } + + pub fn as_non_null_ptr(&self) -> NonNull { + nonnull!(self.as_ptr()) + } + + #[cfg(CPython)] + #[inline(always)] + pub fn from_str_with_hash(buf: &str) -> Self { + let mut obj = PyStrRef::from_str(buf); + obj.set_hash(); + obj + } + + #[cfg(CPython)] + #[inline(always)] + pub fn from_str(buf: &str) -> Self { + if buf.is_empty() { + cold_path!(); + return Self::empty(); + } + #[cfg(not(feature = "avx512"))] + let str_ptr = unsafe { super::scalar::str_impl_kind_scalar(buf) }; + #[cfg(feature = "avx512")] + let str_ptr = unsafe { STR_CREATE_FN(buf) }; + debug_assert!(!str_ptr.is_null()); + Self { + ptr: nonnull!(str_ptr), + } + } + + #[cfg(not(CPython))] + #[inline(always)] + pub fn from_str(buf: &str) -> Self { + if buf.is_empty() { + cold_path!(); + return Self::empty(); + } + let str_ptr = unsafe { + crate::ffi::PyUnicode_FromStringAndSize( + buf.as_ptr().cast::(), + crate::util::usize_to_isize(buf.len()), + ) + }; + debug_assert!(!str_ptr.is_null()); + Self { + ptr: nonnull!(str_ptr), + } + } + + #[cfg(CPython)] + #[allow(unused)] + pub fn hash(&self) -> crate::ffi::Py_hash_t { + unsafe { crate::ffi::PyUnstable_Unicode_GET_CACHED_HASH(self.as_ptr()) } + } + + #[cfg(feature = "inline_str")] + fn set_hash(&mut self) { + unsafe { + let ptr = self.as_ptr().cast::(); + let data_ptr: *mut core::ffi::c_void = + if (*ptr).state & STATE_COMPACT_ASCII == STATE_COMPACT_ASCII { + ptr.offset(1).cast::() + } else { + ptr.cast::() + .offset(1) + .cast::() + }; + #[allow(clippy::cast_possible_wrap)] + let num_bytes = + (*ptr).length * (((*ptr).state & STATE_KIND_MASK) >> STATE_KIND_SHIFT) as isize; + let hash = Py_HashBuffer(data_ptr, num_bytes); + (*ptr).hash = hash; + debug_assert!((*ptr).hash != -1); + } + } + + #[cfg(not(feature = "inline_str"))] + fn set_hash(&mut self) { + unsafe { + let data_ptr = PyUnicode_DATA(self.as_ptr()); + #[allow(clippy::cast_possible_wrap)] + let num_bytes = PyUnicode_KIND(self.as_ptr()) as isize * Py_SIZE(self.as_ptr()); + let hash = Py_HashBuffer(data_ptr, num_bytes); + (*self.as_ptr().cast::()).hash = hash; + debug_assert!(self.hash() != -1); + } + } + + #[inline(always)] + #[cfg(feature = "inline_str")] + pub fn as_str(&self) -> Option<&'static str> { + unsafe { + let op = self.as_ptr(); + if (*op.cast::()).state & STATE_COMPACT_ASCII == STATE_COMPACT_ASCII { + let ptr = op.cast::().offset(1).cast::(); + let len = crate::util::isize_to_usize((*op.cast::()).length); + Some(str_from_slice!(ptr, len)) + } else if (*op.cast::()).state & STATE_COMPACT_ASCII == 0 { + cold_path!(); + to_str_via_ffi(op) + } else if (*op.cast::()).utf8_length != 0 { + let ptr = ((*op.cast::()).utf8).cast::(); + let len = + crate::util::isize_to_usize((*op.cast::()).utf8_length); + Some(str_from_slice!(ptr, len)) + } else { + to_str_via_ffi(op) + } + } + } + + #[inline(always)] + #[cfg(not(feature = "inline_str"))] + pub fn as_str(&self) -> Option<&'static str> { + to_str_via_ffi(self.as_ptr()) + } +} + +#[repr(transparent)] +pub(crate) struct PyStrSubclassRef { + ptr: NonNull, +} + +impl PyStrSubclassRef { + pub unsafe fn from_ptr_unchecked(ptr: *mut PyObject) -> PyStrSubclassRef { + let ob_type = unsafe { crate::ffi::PyObject_Type(ptr) }; + let tp_flags = unsafe { crate::ffi::PyType_GetFlags(ob_type) }; + debug_assert!(!ptr.is_null()); + debug_assert!(!is_class_by_type!(ob_type, STR_TYPE)); + debug_assert!(is_subclass_by_flag!(tp_flags, Py_TPFLAGS_UNICODE_SUBCLASS)); + PyStrSubclassRef { ptr: nonnull!(ptr) } + } + + #[inline] + pub fn as_ptr(&self) -> *mut pyo3_ffi::PyObject { + self.ptr.as_ptr() + } + + #[inline(always)] + pub fn as_str(&self) -> Option<&'static str> { + to_str_via_ffi(self.as_ptr()) + } +} diff --git a/src/ffi/pystrref/pyunicode_new.rs b/src/ffi/pystrref/pyunicode_new.rs new file mode 100644 index 00000000..038c00f3 --- /dev/null +++ b/src/ffi/pystrref/pyunicode_new.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2023-2026) + +use crate::ffi::{PyASCIIObject, PyCompactUnicodeObject, PyObject, PyUnicode_New}; +use crate::util::usize_to_isize; + +macro_rules! validate_str { + ($ptr:expr) => { + #[cfg(all(CPython, not(Py_LIMITED_ABI)))] + debug_assert!(pyo3_ffi::_PyUnicode_CheckConsistency($ptr.cast::(), 1) == 1) + }; +} + +#[inline(never)] +pub(crate) fn pyunicode_ascii(buf: *const u8, num_chars: usize) -> *mut PyObject { + unsafe { + let ptr = PyUnicode_New(usize_to_isize(num_chars), 127); + let data_ptr = ptr.cast::().offset(1).cast::(); + core::ptr::copy_nonoverlapping(buf, data_ptr, num_chars); + core::ptr::write(data_ptr.add(num_chars), 0); + validate_str!(ptr); + ptr.cast::() + } +} + +#[cold] +#[inline(never)] +pub(crate) fn pyunicode_onebyte(buf: &str, num_chars: usize) -> *mut PyObject { + unsafe { + let ptr = PyUnicode_New(usize_to_isize(num_chars), 255); + let mut data_ptr = ptr.cast::().offset(1).cast::(); + for each in buf.chars().fuse() { + core::ptr::write(data_ptr, each as u8); + data_ptr = data_ptr.offset(1); + } + core::ptr::write(data_ptr, 0); + validate_str!(ptr); + ptr.cast::() + } +} + +#[inline(never)] +pub(crate) fn pyunicode_twobyte(buf: &str, num_chars: usize) -> *mut PyObject { + unsafe { + let ptr = PyUnicode_New(usize_to_isize(num_chars), 65535); + let mut data_ptr = ptr.cast::().offset(1).cast::(); + for each in buf.chars().fuse() { + core::ptr::write(data_ptr, each as u16); + data_ptr = data_ptr.offset(1); + } + core::ptr::write(data_ptr, 0); + validate_str!(ptr); + ptr.cast::() + } +} + +#[inline(never)] +pub(crate) fn pyunicode_fourbyte(buf: &str, num_chars: usize) -> *mut PyObject { + unsafe { + let ptr = PyUnicode_New(usize_to_isize(num_chars), 1114111); + let mut data_ptr = ptr.cast::().offset(1).cast::(); + for each in buf.chars().fuse() { + core::ptr::write(data_ptr, each as u32); + data_ptr = data_ptr.offset(1); + } + core::ptr::write(data_ptr, 0); + validate_str!(ptr); + ptr.cast::() + } +} diff --git a/src/ffi/pystrref/scalar.rs b/src/ffi/pystrref/scalar.rs new file mode 100644 index 00000000..f6a472e5 --- /dev/null +++ b/src/ffi/pystrref/scalar.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2024-2026) + +use super::pyunicode_new::{ + pyunicode_ascii, pyunicode_fourbyte, pyunicode_onebyte, pyunicode_twobyte, +}; + +#[inline(never)] +pub(crate) unsafe fn str_impl_kind_scalar(buf: &str) -> *mut crate::ffi::PyObject { + let num_chars = bytecount::num_chars(buf.as_bytes()); + if buf.len() == num_chars { + return pyunicode_ascii(buf.as_ptr(), num_chars); + } + unsafe { + let len = buf.len(); + assume!(len > 0); + + if *(buf.as_bytes().as_ptr()) > 239 { + cold_path!(); + return pyunicode_fourbyte(buf, num_chars); + } + + let sptr = buf.as_bytes().as_ptr(); + + let mut is_four = false; + let mut not_latin = false; + for i in 0..len { + is_four |= *sptr.add(i) > 239; + not_latin |= *sptr.add(i) > 195; + } + if is_four { + pyunicode_fourbyte(buf, num_chars) + } else if not_latin { + pyunicode_twobyte(buf, num_chars) + } else { + pyunicode_onebyte(buf, num_chars) + } + } +} diff --git a/src/ffi/pytimeref.rs b/src/ffi/pytimeref.rs new file mode 100644 index 00000000..57faa722 --- /dev/null +++ b/src/ffi/pytimeref.rs @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +use crate::ffi::{ + PyDateTime_TIME_GET_HOUR, PyDateTime_TIME_GET_MICROSECOND, PyDateTime_TIME_GET_MINUTE, + PyDateTime_TIME_GET_SECOND, PyDateTime_Time, PyObject, +}; + +#[derive(Clone)] +#[repr(transparent)] +pub(crate) struct PyTimeRef { + ptr: core::ptr::NonNull, +} + +unsafe impl Send for PyTimeRef {} +unsafe impl Sync for PyTimeRef {} + +impl PartialEq for PyTimeRef { + fn eq(&self, other: &Self) -> bool { + self.ptr == other.ptr + } +} + +impl PyTimeRef { + #[inline] + pub(crate) unsafe fn from_ptr_unchecked(ptr: *mut PyObject) -> Self { + unsafe { + debug_assert!(!ptr.is_null()); + debug_assert!(crate::ffi::PyObject_Type(ptr) == crate::typeref::TIME_TYPE); + Self { + ptr: core::ptr::NonNull::new_unchecked(ptr), + } + } + } + + #[inline] + #[allow(unused)] + pub fn as_ptr(&self) -> *mut PyObject { + self.ptr.as_ptr() + } + + #[inline] + #[allow(unused)] + pub fn as_non_null_ptr(&self) -> core::ptr::NonNull { + self.ptr + } + + #[cfg(CPython)] + #[inline] + pub fn has_tz(&self) -> bool { + unsafe { (*self.ptr.as_ptr().cast::()).hastzinfo == 1 } + } + + #[cfg(not(CPython))] + #[inline] + pub fn has_tz(&self) -> bool { + unimplemented!() + } + + #[inline] + pub fn hour(&self) -> u8 { + unsafe { PyDateTime_TIME_GET_HOUR(self.ptr.as_ptr()).cast_unsigned() as u8 } + } + + #[inline] + pub fn minute(&self) -> u8 { + unsafe { PyDateTime_TIME_GET_MINUTE(self.ptr.as_ptr()).cast_unsigned() as u8 } + } + + #[inline] + pub fn second(&self) -> u8 { + unsafe { PyDateTime_TIME_GET_SECOND(self.ptr.as_ptr()).cast_unsigned() as u8 } + } + + #[inline] + pub fn microsecond(&self) -> u32 { + unsafe { PyDateTime_TIME_GET_MICROSECOND(self.ptr.as_ptr()).cast_unsigned() } + } +} diff --git a/src/ffi/pytupleref.rs b/src/ffi/pytupleref.rs new file mode 100644 index 00000000..c6dbd11f --- /dev/null +++ b/src/ffi/pytupleref.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +#[derive(Clone)] +#[repr(transparent)] +pub(crate) struct PyTupleRef { + ptr: core::ptr::NonNull, +} + +unsafe impl Send for PyTupleRef {} +unsafe impl Sync for PyTupleRef {} + +impl PartialEq for PyTupleRef { + fn eq(&self, other: &Self) -> bool { + self.ptr == other.ptr + } +} + +impl PyTupleRef { + #[inline] + pub fn with_capacity(cap: usize) -> Self { + unsafe { + let ptr = crate::ffi::PyTuple_New(crate::util::usize_to_isize(cap)); + debug_assert!(!ptr.is_null()); + Self { ptr: nonnull!(ptr) } + } + } + + #[inline] + pub unsafe fn from_ptr_unchecked(ptr: *mut pyo3_ffi::PyObject) -> Self { + unsafe { + debug_assert!(!ptr.is_null()); + debug_assert!( + is_type!(crate::ffi::PyObject_Type(ptr), crate::typeref::TUPLE_TYPE) + || is_subclass_by_flag!( + crate::ffi::PyType_GetFlags(crate::ffi::PyObject_Type(ptr)), + Py_TPFLAGS_TUPLE_SUBCLASS + ) + ); + Self { + ptr: core::ptr::NonNull::new_unchecked(ptr), + } + } + } + + #[inline] + #[allow(unused)] + pub fn as_ptr(&self) -> *mut pyo3_ffi::PyObject { + self.ptr.as_ptr() + } + + #[inline] + #[allow(unused)] + pub fn as_non_null_ptr(&self) -> core::ptr::NonNull { + self.ptr + } + + pub fn get(&self, i: usize) -> *mut pyo3_ffi::PyObject { + unsafe { crate::ffi::PyTuple_GET_ITEM(self.ptr.as_ptr(), crate::util::usize_to_isize(i)) } + } + + #[inline] + pub fn set(&mut self, i: usize, val: *mut pyo3_ffi::PyObject) { + unsafe { + crate::ffi::PyTuple_SET_ITEM(self.as_ptr(), crate::util::usize_to_isize(i), val); + } + } + + #[inline] + pub fn len(&self) -> usize { + unsafe { crate::util::isize_to_usize(super::Py_SIZE(self.ptr.as_ptr())) } + } +} diff --git a/src/ffi/pytype.rs b/src/ffi/pytype.rs deleted file mode 100644 index 3bd15dcf..00000000 --- a/src/ffi/pytype.rs +++ /dev/null @@ -1,13 +0,0 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -use std::os::raw::c_char; - -#[repr(C)] -#[derive(Debug, Copy, Clone)] -pub struct PyTypeObject { - pub ob_refcnt: pyo3_ffi::Py_ssize_t, - pub ob_type: *mut pyo3_ffi::PyTypeObject, - pub ma_used: pyo3_ffi::Py_ssize_t, - pub tp_name: *const c_char, - // ... -} diff --git a/src/ffi/pyuuidref.rs b/src/ffi/pyuuidref.rs new file mode 100644 index 00000000..dc1bb1bc --- /dev/null +++ b/src/ffi/pyuuidref.rs @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +use super::{Py_DECREF, PyLong_AsByteArray, PyLongObject, PyObject, PyObject_GetAttr}; +use crate::typeref::INT_ATTR_STR; + +#[derive(Clone)] +#[repr(transparent)] +pub(crate) struct PyUuidRef { + ptr: core::ptr::NonNull, +} + +unsafe impl Send for PyUuidRef {} +unsafe impl Sync for PyUuidRef {} + +impl PartialEq for PyUuidRef { + fn eq(&self, other: &Self) -> bool { + self.ptr == other.ptr + } +} + +impl PyUuidRef { + #[inline] + pub(crate) unsafe fn from_ptr_unchecked(ptr: *mut PyObject) -> Self { + unsafe { + debug_assert!(!ptr.is_null()); + debug_assert!(crate::ffi::PyObject_Type(ptr) == crate::typeref::UUID_TYPE); + Self { + ptr: core::ptr::NonNull::new_unchecked(ptr), + } + } + } + + #[inline] + pub(crate) fn value(&self, buffer: &mut [u8; 16]) { + unsafe { + let py_int = PyObject_GetAttr(self.ptr.as_ptr(), INT_ATTR_STR); + PyLong_AsByteArray(py_int.cast::(), buffer as *mut u8, 16, 0, 0); + Py_DECREF(py_int); + } + } +} diff --git a/src/ffi/utf8.rs b/src/ffi/utf8.rs new file mode 100644 index 00000000..b7fc79aa --- /dev/null +++ b/src/ffi/utf8.rs @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2021-2026) + +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +pub(crate) fn is_valid_utf8(buf: &[u8]) -> bool { + if std::is_x86_feature_detected!("avx2") { + unsafe { simdutf8::basic::imp::x86::avx2::validate_utf8(buf).is_ok() } + } else { + encoding_rs::Encoding::utf8_valid_up_to(buf) == buf.len() + } +} + +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +pub(crate) fn is_valid_utf8(buf: &[u8]) -> bool { + simdutf8::basic::from_utf8(buf).is_ok() +} + +#[cfg(target_arch = "aarch64")] +pub(crate) fn is_valid_utf8(buf: &[u8]) -> bool { + unsafe { simdutf8::basic::imp::aarch64::neon::validate_utf8(buf).is_ok() } +} + +#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] +pub(crate) fn is_valid_utf8(buf: &[u8]) -> bool { + std::str::from_utf8(buf).is_ok() +} diff --git a/src/lib.rs b/src/lib.rs index 5f4f9155..92280e9e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,404 +1,322 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -#![cfg_attr(feature = "unstable-simd", feature(core_intrinsics))] -#![cfg_attr(feature = "unstable-simd", feature(optimize_attribute))] +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2018-2026) + +#![cfg_attr(feature = "generic_simd", feature(portable_simd))] +#![cfg_attr(feature = "optimize", feature(optimize_attribute))] +#![allow(unused_features)] // portable_simd on universal2 cross-compile +#![allow(stable_features)] +#![allow(static_mut_refs)] #![allow(unused_unsafe)] +#![warn(clippy::complexity)] +#![warn(clippy::correctness)] +#![warn(clippy::perf)] +#![warn(clippy::style)] +#![warn(clippy::suspicious)] +#![allow(clippy::explicit_iter_loop)] +#![allow(clippy::inline_always)] #![allow(clippy::missing_safety_doc)] #![allow(clippy::redundant_field_names)] #![allow(clippy::upper_case_acronyms)] #![allow(clippy::zero_prefixed_literal)] -#![allow(non_camel_case_types)] +#![warn(clippy::borrow_as_ptr)] +#![warn(clippy::cast_possible_wrap)] +#![warn(clippy::cast_ptr_alignment)] +#![warn(clippy::cast_sign_loss)] +#![warn(clippy::elidable_lifetime_names)] +#![warn(clippy::ptr_arg)] +#![warn(clippy::ptr_as_ptr)] +#![warn(clippy::ptr_cast_constness)] +#![warn(clippy::ptr_eq)] +#![warn(clippy::redundant_allocation)] +#![warn(clippy::redundant_clone)] +#![warn(clippy::redundant_locals)] +#![warn(clippy::redundant_slicing)] +#![warn(clippy::semicolon_inside_block)] +#![warn(clippy::size_of_ref)] +#![warn(clippy::std_instead_of_core)] +#![warn(clippy::trivially_copy_pass_by_ref)] +#![warn(clippy::unnecessary_semicolon)] +#![warn(clippy::unnecessary_wraps)] +#![warn(clippy::zero_ptr)] + +#[cfg(feature = "unwind")] +extern crate unwinding; #[macro_use] mod util; +mod alloc; mod deserialize; -mod error; +mod exception; mod ffi; mod opt; mod serialize; mod typeref; -mod unicode; - -#[cfg(feature = "yyjson")] -mod yyjson; - -use pyo3_ffi::*; -use std::borrow::Cow; -use std::os::raw::c_char; -use std::os::raw::c_int; -use std::os::raw::c_void; - -#[allow(unused_imports)] -use core::ptr::{null, null_mut, NonNull}; -#[cfg(Py_3_10)] +use core::ffi::{c_char, c_int, c_void}; +use core::ptr::{NonNull, null, null_mut}; + +use crate::deserialize::deserialize; +use crate::exception::{ + raise_dumps_exception_dynamic, raise_dumps_exception_fixed, raise_loads_exception, +}; +use crate::ffi::{ + METH_KEYWORDS, METH_O, Py_SIZE, Py_ssize_t, PyCFunction_NewEx, PyIntRef, PyMethodDef, + PyMethodDefPointer, PyModuleDef, PyModuleDef_HEAD_INIT, PyModuleDef_Init, PyModuleDef_Slot, + PyNoneRef, PyObject, PyTupleRef, PyUnicode_FromStringAndSize, PyUnicode_InternFromString, + PyVectorcall_NARGS, +}; +use crate::serialize::serialize; +use crate::util::{isize_to_usize, usize_to_isize}; + +#[cfg(Py_3_13)] macro_rules! add { ($mptr:expr, $name:expr, $obj:expr) => { - PyModule_AddObjectRef($mptr, $name.as_ptr() as *const c_char, $obj); + crate::ffi::PyModule_Add($mptr, $name.as_ptr(), $obj); }; } -#[cfg(not(Py_3_10))] +#[cfg(not(Py_3_13))] macro_rules! add { ($mptr:expr, $name:expr, $obj:expr) => { - PyModule_AddObject($mptr, $name.as_ptr() as *const c_char, $obj); + crate::ffi::PyModule_AddObjectRef($mptr, $name.as_ptr(), $obj); }; } macro_rules! opt { ($mptr:expr, $name:expr, $opt:expr) => { #[cfg(all(not(target_os = "windows"), target_pointer_width = "64"))] - PyModule_AddIntConstant($mptr, $name.as_ptr() as *const c_char, $opt as i64); + crate::ffi::PyModule_AddIntConstant($mptr, $name.as_ptr(), i64::from($opt)); #[cfg(all(not(target_os = "windows"), target_pointer_width = "32"))] - PyModule_AddIntConstant($mptr, $name.as_ptr() as *const c_char, $opt as i32); + crate::ffi::PyModule_AddIntConstant($mptr, $name.as_ptr(), $opt as i32); #[cfg(target_os = "windows")] - PyModule_AddIntConstant($mptr, $name.as_ptr() as *const c_char, $opt as i32); + crate::ffi::PyModule_AddIntConstant($mptr, $name.as_ptr(), $opt as i32); }; } #[allow(non_snake_case)] -#[no_mangle] +#[unsafe(no_mangle)] #[cold] -#[cfg_attr(feature = "unstable-simd", optimize(size))] -pub unsafe extern "C" fn orjson_init_exec(mptr: *mut PyObject) -> c_int { - typeref::init_typerefs(); - { - let version = env!("CARGO_PKG_VERSION"); - let pyversion = - PyUnicode_FromStringAndSize(version.as_ptr() as *const c_char, version.len() as isize); - add!(mptr, "__version__\0", pyversion); - } - { - let dumps_doc = - "dumps(obj, /, default=None, option=None)\n--\n\nSerialize Python objects to JSON.\0"; - - let wrapped_dumps: PyMethodDef; +#[cfg_attr(feature = "optimize", optimize(size))] +pub(crate) unsafe extern "C" fn orjson_init_exec(mptr: *mut PyObject) -> c_int { + unsafe { + typeref::init_typerefs(); - #[cfg(Py_3_8)] { - wrapped_dumps = PyMethodDef { - ml_name: "dumps\0".as_ptr() as *const c_char, - ml_meth: PyMethodDefPointer { - _PyCFunctionFastWithKeywords: dumps, - }, - ml_flags: pyo3_ffi::METH_FASTCALL | METH_KEYWORDS, - ml_doc: dumps_doc.as_ptr() as *const c_char, - }; + let version = env!("CARGO_PKG_VERSION"); + let pyversion = PyUnicode_FromStringAndSize( + version.as_ptr().cast::(), + usize_to_isize(version.len()), + ); + add!(mptr, c"__version__", pyversion); } - #[cfg(not(Py_3_8))] + { - wrapped_dumps = PyMethodDef { - ml_name: "dumps\0".as_ptr() as *const c_char, + let dumps_doc = c"dumps(obj, /, default=None, option=None)\n--\n\nSerialize Python objects to JSON."; + + let wrapped_dumps = Box::new(PyMethodDef { + ml_name: c"dumps".as_ptr(), ml_meth: PyMethodDefPointer { - PyCFunctionWithKeywords: dumps, + PyCFunctionFastWithKeywords: dumps, }, - ml_flags: METH_VARARGS | METH_KEYWORDS, - ml_doc: dumps_doc.as_ptr() as *const c_char, - }; + ml_flags: crate::ffi::METH_FASTCALL | METH_KEYWORDS, + ml_doc: dumps_doc.as_ptr(), + }); + + let func = PyCFunction_NewEx( + Box::into_raw(wrapped_dumps), + null_mut(), + PyUnicode_InternFromString(c"orjson".as_ptr()), + ); + add!(mptr, c"dumps", func); } - let func = PyCFunction_NewEx( - Box::into_raw(Box::new(wrapped_dumps)), - null_mut(), - PyUnicode_InternFromString("orjson\0".as_ptr() as *const c_char), - ); - add!(mptr, "dumps\0", func); - } - - { - let loads_doc = "loads(obj, /)\n--\n\nDeserialize JSON to Python objects.\0"; + { + let loads_doc = c"loads(obj, /)\n--\n\nDeserialize JSON to Python objects."; + + let wrapped_loads = Box::new(PyMethodDef { + ml_name: c"loads".as_ptr(), + ml_meth: PyMethodDefPointer { PyCFunction: loads }, + ml_flags: METH_O, + ml_doc: loads_doc.as_ptr(), + }); + let func = PyCFunction_NewEx( + Box::into_raw(wrapped_loads), + null_mut(), + PyUnicode_InternFromString(c"orjson".as_ptr()), + ); + add!(mptr, c"loads", func); + } - let wrapped_loads = PyMethodDef { - ml_name: "loads\0".as_ptr() as *const c_char, - ml_meth: PyMethodDefPointer { PyCFunction: loads }, - ml_flags: METH_O, - ml_doc: loads_doc.as_ptr() as *const c_char, - }; - let func = PyCFunction_NewEx( - Box::into_raw(Box::new(wrapped_loads)), - null_mut(), - PyUnicode_InternFromString("orjson\0".as_ptr() as *const c_char), + add!(mptr, c"Fragment", typeref::FRAGMENT_TYPE.cast::()); + + opt!(mptr, c"OPT_APPEND_NEWLINE", opt::APPEND_NEWLINE); + opt!(mptr, c"OPT_INDENT_2", opt::INDENT_2); + opt!(mptr, c"OPT_NAIVE_UTC", opt::NAIVE_UTC); + opt!(mptr, c"OPT_NON_STR_KEYS", opt::NON_STR_KEYS); + opt!(mptr, c"OPT_OMIT_MICROSECONDS", opt::OMIT_MICROSECONDS); + opt!( + mptr, + c"OPT_PASSTHROUGH_DATACLASS", + opt::PASSTHROUGH_DATACLASS ); - add!(mptr, "loads\0", func); + opt!(mptr, c"OPT_PASSTHROUGH_DATETIME", opt::PASSTHROUGH_DATETIME); + opt!(mptr, c"OPT_PASSTHROUGH_SUBCLASS", opt::PASSTHROUGH_SUBCLASS); + opt!(mptr, c"OPT_SERIALIZE_DATACLASS", opt::SERIALIZE_DATACLASS); + opt!(mptr, c"OPT_SERIALIZE_NUMPY", opt::SERIALIZE_NUMPY); + opt!(mptr, c"OPT_SERIALIZE_UUID", opt::SERIALIZE_UUID); + opt!(mptr, c"OPT_SORT_KEYS", opt::SORT_KEYS); + opt!(mptr, c"OPT_STRICT_INTEGER", opt::STRICT_INTEGER); + opt!(mptr, c"OPT_UTC_Z", opt::UTC_Z); + + add!(mptr, c"JSONDecodeError", typeref::JsonDecodeError); + add!(mptr, c"JSONEncodeError", typeref::JsonEncodeError); + + 0 } - - opt!(mptr, "OPT_APPEND_NEWLINE\0", opt::APPEND_NEWLINE); - opt!(mptr, "OPT_INDENT_2\0", opt::INDENT_2); - opt!(mptr, "OPT_NAIVE_UTC\0", opt::NAIVE_UTC); - opt!(mptr, "OPT_NON_STR_KEYS\0", opt::NON_STR_KEYS); - opt!(mptr, "OPT_OMIT_MICROSECONDS\0", opt::OMIT_MICROSECONDS); - opt!( - mptr, - "OPT_PASSTHROUGH_DATACLASS\0", - opt::PASSTHROUGH_DATACLASS - ); - opt!( - mptr, - "OPT_PASSTHROUGH_DATETIME\0", - opt::PASSTHROUGH_DATETIME - ); - opt!( - mptr, - "OPT_PASSTHROUGH_SUBCLASS\0", - opt::PASSTHROUGH_SUBCLASS - ); - opt!(mptr, "OPT_SERIALIZE_DATACLASS\0", opt::SERIALIZE_DATACLASS); - opt!(mptr, "OPT_SERIALIZE_NUMPY\0", opt::SERIALIZE_NUMPY); - opt!(mptr, "OPT_SERIALIZE_UUID\0", opt::SERIALIZE_UUID); - opt!(mptr, "OPT_SORT_KEYS\0", opt::SORT_KEYS); - opt!(mptr, "OPT_STRICT_INTEGER\0", opt::STRICT_INTEGER); - opt!(mptr, "OPT_UTC_Z\0", opt::UTC_Z); - - add!(mptr, "JSONDecodeError\0", typeref::JsonDecodeError); - add!(mptr, "JSONEncodeError\0", typeref::JsonEncodeError); - - // maturin>=0.11.0 creates a python package that imports *, hiding dunder by default - let all: [&str; 20] = [ - "__all__\0", - "__version__\0", - "dumps\0", - "JSONDecodeError\0", - "JSONEncodeError\0", - "loads\0", - "OPT_APPEND_NEWLINE\0", - "OPT_INDENT_2\0", - "OPT_NAIVE_UTC\0", - "OPT_NON_STR_KEYS\0", - "OPT_OMIT_MICROSECONDS\0", - "OPT_PASSTHROUGH_DATACLASS\0", - "OPT_PASSTHROUGH_DATETIME\0", - "OPT_PASSTHROUGH_SUBCLASS\0", - "OPT_SERIALIZE_DATACLASS\0", - "OPT_SERIALIZE_NUMPY\0", - "OPT_SERIALIZE_UUID\0", - "OPT_SORT_KEYS\0", - "OPT_STRICT_INTEGER\0", - "OPT_UTC_Z\0", - ]; - - let pyall = PyTuple_New(all.len() as isize); - for (i, obj) in all.iter().enumerate() { - PyTuple_SET_ITEM( - pyall, - i as isize, - PyUnicode_InternFromString(obj.as_ptr() as *const c_char), - ) - } - - add!(mptr, "__all__\0", pyall); - 0 } #[allow(non_snake_case)] -#[no_mangle] +#[unsafe(no_mangle)] #[cold] -#[cfg_attr(feature = "unstable-simd", optimize(size))] -pub unsafe extern "C" fn PyInit_orjson() -> *mut PyModuleDef { - let mod_slots: Box<[PyModuleDef_Slot; 2]> = Box::new([ - PyModuleDef_Slot { - slot: Py_mod_exec, - value: orjson_init_exec as *mut c_void, - }, - PyModuleDef_Slot { - slot: 0, - value: null_mut(), - }, - ]); - - let init = Box::new(PyModuleDef { - m_base: PyModuleDef_HEAD_INIT, - m_name: "orjson\0".as_ptr() as *const c_char, - m_doc: null(), - m_size: 0, - m_methods: null_mut(), - m_slots: Box::into_raw(mod_slots) as *mut PyModuleDef_Slot, - m_traverse: None, - m_clear: None, - m_free: None, - }); - let init_ptr = Box::into_raw(init); - PyModuleDef_Init(init_ptr); - init_ptr +#[cfg_attr(feature = "optimize", optimize(size))] +pub(crate) unsafe extern "C" fn PyInit_orjson() -> *mut PyModuleDef { + unsafe { + let mod_slots = Box::new([ + PyModuleDef_Slot { + slot: crate::ffi::Py_mod_exec, + #[allow(clippy::fn_to_numeric_cast_any, clippy::as_conversions)] + value: orjson_init_exec as *mut c_void, + }, + #[cfg(Py_3_12)] + PyModuleDef_Slot { + slot: crate::ffi::Py_mod_multiple_interpreters, + value: crate::ffi::Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED, + }, + #[cfg(Py_3_13)] + PyModuleDef_Slot { + slot: crate::ffi::Py_mod_gil, + value: crate::ffi::Py_MOD_GIL_USED, + }, + PyModuleDef_Slot { + slot: 0, + value: null_mut(), + }, + ]); + + let init = Box::new(PyModuleDef { + m_base: PyModuleDef_HEAD_INIT, + m_name: c"orjson".as_ptr(), + m_doc: null(), + m_size: 0, + m_methods: null_mut(), + m_slots: Box::into_raw(mod_slots).cast::(), + m_traverse: None, + m_clear: None, + m_free: None, + }); + let init_ptr = Box::into_raw(init); + PyModuleDef_Init(init_ptr); + init_ptr + } } -#[cold] -#[inline(never)] -#[cfg_attr(feature = "unstable-simd", optimize(size))] -fn raise_loads_exception(err: deserialize::DeserializeError) -> *mut PyObject { - let pos = err.pos(); - let msg = err.message; - let doc = err.data; - unsafe { - let err_msg = - PyUnicode_FromStringAndSize(msg.as_ptr() as *const c_char, msg.len() as isize); - let args = PyTuple_New(3); - let doc = PyUnicode_FromStringAndSize(doc.as_ptr() as *const c_char, doc.len() as isize); - let pos = PyLong_FromLongLong(pos); - PyTuple_SET_ITEM(args, 0, err_msg); - PyTuple_SET_ITEM(args, 1, doc); - PyTuple_SET_ITEM(args, 2, pos); - PyErr_SetObject(typeref::JsonDecodeError, args); - Py_DECREF(args); - }; - null_mut() +#[unsafe(no_mangle)] +pub(crate) unsafe extern "C" fn loads(_self: *mut PyObject, obj: *mut PyObject) -> *mut PyObject { + deserialize(obj).map_or_else(raise_loads_exception, NonNull::as_ptr) } -#[cold] -#[inline(never)] -#[cfg_attr(feature = "unstable-simd", optimize(size))] -fn raise_dumps_exception(msg: Cow) -> *mut PyObject { - unsafe { - let err_msg = - PyUnicode_FromStringAndSize(msg.as_ptr() as *const c_char, msg.len() as isize); - PyErr_SetObject(typeref::JsonEncodeError, err_msg); - Py_DECREF(err_msg); +#[cfg(CPython)] +macro_rules! matches_kwarg { + ($val:expr, $ref:expr) => { + core::ptr::eq($val, $ref) }; - null_mut() } -#[no_mangle] -pub unsafe extern "C" fn loads(_self: *mut PyObject, obj: *mut PyObject) -> *mut PyObject { - match crate::deserialize::deserialize(obj) { - Ok(val) => val.as_ptr(), - Err(err) => raise_loads_exception(err), - } +#[cfg(not(CPython))] +macro_rules! matches_kwarg { + ($val:expr, $ref:expr) => { + crate::ffi::PyObject_Hash($val) == crate::ffi::PyObject_Hash($ref) + }; } -#[cfg(Py_3_8)] -#[no_mangle] -pub unsafe extern "C" fn dumps( +#[unsafe(no_mangle)] +pub(crate) unsafe extern "C" fn dumps( _self: *mut PyObject, args: *const *mut PyObject, nargs: Py_ssize_t, kwnames: *mut PyObject, ) -> *mut PyObject { - let mut default: Option> = None; - let mut optsptr: Option> = None; - - let num_args = PyVectorcall_NARGS(nargs as usize); - if unlikely!(num_args == 0) { - return raise_dumps_exception(Cow::Borrowed( - "dumps() missing 1 required positional argument: 'obj'", - )); - } - if num_args & 2 == 2 { - default = Some(NonNull::new_unchecked(*args.offset(1))); - } - if num_args & 3 == 3 { - optsptr = Some(NonNull::new_unchecked(*args.offset(2))); - } - if !kwnames.is_null() { - for i in 0..=PyTuple_GET_SIZE(kwnames) - 1 { - let arg = PyTuple_GET_ITEM(kwnames, i as Py_ssize_t); - if arg == typeref::DEFAULT { - if unlikely!(num_args & 2 == 2) { - return raise_dumps_exception(Cow::Borrowed( - "dumps() got multiple values for argument: 'default'", - )); - } - default = Some(NonNull::new_unchecked(*args.offset(num_args + i))); - } else if arg == typeref::OPTION { - if unlikely!(num_args & 3 == 3) { - return raise_dumps_exception(Cow::Borrowed( - "dumps() got multiple values for argument: 'option'", - )); - } - optsptr = Some(NonNull::new_unchecked(*args.offset(num_args + i))); - } else { - return raise_dumps_exception(Cow::Borrowed( - "dumps() got an unexpected keyword argument", - )); - } + unsafe { + let mut default: Option> = None; + let mut optsptr: Option> = None; + + let num_args = PyVectorcall_NARGS(isize_to_usize(nargs)); + if num_args == 0 { + cold_path!(); + return raise_dumps_exception_fixed( + "dumps() missing 1 required positional argument: 'obj'", + ); } - } - - let mut optsbits: i32 = 0; - if let Some(opts) = optsptr { - if (*opts.as_ptr()).ob_type != typeref::INT_TYPE { - return raise_dumps_exception(Cow::Borrowed("Invalid opts")); + if num_args & 2 == 2 { + default = Some(NonNull::new_unchecked(*args.offset(1))); } - optsbits = PyLong_AsLong(optsptr.unwrap().as_ptr()) as i32; - if !(0..=opt::MAX_OPT).contains(&optsbits) { - return raise_dumps_exception(Cow::Borrowed("Invalid opts")); + if num_args & 3 == 3 { + optsptr = Some(NonNull::new_unchecked(*args.offset(2))); } - } - - match crate::serialize::serialize(*args, default, optsbits as opt::Opt) { - Ok(val) => val.as_ptr(), - Err(err) => raise_dumps_exception(Cow::Borrowed(&err)), - } -} - -#[cfg(not(Py_3_8))] -#[no_mangle] -pub unsafe extern "C" fn dumps( - _self: *mut PyObject, - args: *mut PyObject, - kwds: *mut PyObject, -) -> *mut PyObject { - let mut default: Option> = None; - let mut optsptr: Option> = None; - - let obj = PyTuple_GET_ITEM(args, 0); - - let num_args = PyTuple_GET_SIZE(args); - if unlikely!(num_args == 0) { - return raise_dumps_exception(Cow::Borrowed( - "dumps() missing 1 required positional argument: 'obj'", - )); - } - if num_args & 2 == 2 { - default = Some(NonNull::new_unchecked(PyTuple_GET_ITEM(args, 1))); - } - if num_args & 3 == 3 { - optsptr = Some(NonNull::new_unchecked(PyTuple_GET_ITEM(args, 2))); - } - - if !kwds.is_null() { - let len = unsafe { crate::ffi::PyDict_GET_SIZE(kwds) }; - let mut pos = 0isize; - let mut arg: *mut PyObject = null_mut(); - let mut val: *mut PyObject = null_mut(); - for _ in 0..=len.saturating_sub(1) { - unsafe { _PyDict_Next(kwds, &mut pos, &mut arg, &mut val, null_mut()) }; - if arg == typeref::DEFAULT { - if unlikely!(num_args & 2 == 2) { - return raise_dumps_exception(Cow::Borrowed( - "dumps() got multiple values for argument: 'default'", - )); + if !kwnames.is_null() { + cold_path!(); + let kwob = PyTupleRef::from_ptr_unchecked(kwnames); + for i in 0..=Py_SIZE(kwnames).saturating_sub(1) { + let arg = kwob.get(i.cast_unsigned()); + if matches_kwarg!(arg, typeref::OPTION) { + if num_args & 3 == 3 { + cold_path!(); + return raise_dumps_exception_fixed( + "dumps() got multiple values for argument: 'option'", + ); + } + optsptr = Some(NonNull::new_unchecked(*args.offset(num_args + i))); + } else if matches_kwarg!(arg, typeref::DEFAULT) { + if num_args & 2 == 2 { + cold_path!(); + return raise_dumps_exception_fixed( + "dumps() got multiple values for argument: 'default'", + ); + } + default = Some(NonNull::new_unchecked(*args.offset(num_args + i))); + } else { + return raise_dumps_exception_fixed( + "dumps() got an unexpected keyword argument", + ); } - default = Some(NonNull::new_unchecked(val)); - } else if arg == typeref::OPTION { - if unlikely!(num_args & 3 == 3) { - return raise_dumps_exception(Cow::Borrowed( - "dumps() got multiple values for argument: 'option'", - )); - } - optsptr = Some(NonNull::new_unchecked(val)); - } else if arg.is_null() { - break; - } else { - return raise_dumps_exception(Cow::Borrowed( - "dumps() got an unexpected keyword argument", - )); } } - } - let mut optsbits: i32 = 0; - if let Some(opts) = optsptr { - if (*opts.as_ptr()).ob_type != typeref::INT_TYPE { - return raise_dumps_exception(Cow::Borrowed("Invalid opts")); - } - optsbits = PyLong_AsLong(optsptr.unwrap().as_ptr()) as i32; - if optsbits < 0 || optsbits > opt::MAX_OPT { - return raise_dumps_exception(Cow::Borrowed("Invalid opts")); + let mut opts = 0 as opt::Opt; + if let Some(tmp) = optsptr { + cold_path!(); + match PyIntRef::from_ptr(tmp.as_ptr()) { + Ok(val) => match val.as_opt() { + Ok(opt) => { + opts = opt; + } + Err(_) => { + return raise_dumps_exception_fixed("Invalid opts"); + } + }, + Err(_) => { + if !core::ptr::eq(tmp.as_ptr(), PyNoneRef::none().as_ptr()) { + cold_path!(); + return raise_dumps_exception_fixed("Invalid opts"); + } + } + } } - } - match crate::serialize::serialize(obj, default, optsbits as opt::Opt) { - Ok(val) => val.as_ptr(), - Err(err) => raise_dumps_exception(Cow::Owned(err)), + serialize(*args, default, opts).map_or_else( + |err| raise_dumps_exception_dynamic(err.as_str()), + NonNull::as_ptr, + ) } } diff --git a/src/opt.rs b/src/opt.rs index 06963d62..7a36fa97 100644 --- a/src/opt.rs +++ b/src/opt.rs @@ -1,30 +1,32 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2020-2025) -pub type Opt = u16; +pub(crate) type Opt = u32; -pub const INDENT_2: Opt = 1; -pub const NAIVE_UTC: Opt = 1 << 1; -pub const NON_STR_KEYS: Opt = 1 << 2; -pub const OMIT_MICROSECONDS: Opt = 1 << 3; -pub const SERIALIZE_NUMPY: Opt = 1 << 4; -pub const SORT_KEYS: Opt = 1 << 5; -pub const STRICT_INTEGER: Opt = 1 << 6; -pub const UTC_Z: Opt = 1 << 7; -pub const PASSTHROUGH_SUBCLASS: Opt = 1 << 8; -pub const PASSTHROUGH_DATETIME: Opt = 1 << 9; -pub const APPEND_NEWLINE: Opt = 1 << 10; -pub const PASSTHROUGH_DATACLASS: Opt = 1 << 11; +pub(crate) const INDENT_2: Opt = 1; +pub(crate) const NAIVE_UTC: Opt = 1 << 1; +pub(crate) const NON_STR_KEYS: Opt = 1 << 2; +pub(crate) const OMIT_MICROSECONDS: Opt = 1 << 3; +pub(crate) const SERIALIZE_NUMPY: Opt = 1 << 4; +pub(crate) const SORT_KEYS: Opt = 1 << 5; +pub(crate) const STRICT_INTEGER: Opt = 1 << 6; +pub(crate) const UTC_Z: Opt = 1 << 7; +pub(crate) const PASSTHROUGH_SUBCLASS: Opt = 1 << 8; +pub(crate) const PASSTHROUGH_DATETIME: Opt = 1 << 9; +pub(crate) const APPEND_NEWLINE: Opt = 1 << 10; +pub(crate) const PASSTHROUGH_DATACLASS: Opt = 1 << 11; // deprecated -pub const SERIALIZE_DATACLASS: Opt = 0; -pub const SERIALIZE_UUID: Opt = 0; +pub(crate) const SERIALIZE_DATACLASS: Opt = 0; +pub(crate) const SERIALIZE_UUID: Opt = 0; -pub const SORT_OR_NON_STR_KEYS: Opt = SORT_KEYS | NON_STR_KEYS; +pub(crate) const SORT_OR_NON_STR_KEYS: Opt = SORT_KEYS | NON_STR_KEYS; -pub const NOT_PASSTHROUGH: Opt = +pub(crate) const NOT_PASSTHROUGH: Opt = !(PASSTHROUGH_DATETIME | PASSTHROUGH_DATACLASS | PASSTHROUGH_SUBCLASS); -pub const MAX_OPT: i32 = (APPEND_NEWLINE +#[allow(clippy::cast_possible_wrap)] +pub(crate) const MAX_OPT: i32 = (APPEND_NEWLINE | INDENT_2 | NAIVE_UTC | NON_STR_KEYS diff --git a/src/serialize/dataclass.rs b/src/serialize/dataclass.rs deleted file mode 100644 index 2d57e15c..00000000 --- a/src/serialize/dataclass.rs +++ /dev/null @@ -1,170 +0,0 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -use crate::ffi::PyDict_GET_SIZE; -use crate::opt::*; -use crate::serialize::error::*; -use crate::serialize::serializer::*; -use crate::typeref::*; -use crate::unicode::*; - -use serde::ser::{Serialize, SerializeMap, Serializer}; - -use std::ptr::addr_of_mut; -use std::ptr::NonNull; - -pub struct DataclassFastSerializer { - dict: *mut pyo3_ffi::PyObject, - opts: Opt, - default_calls: u8, - recursion: u8, - default: Option>, -} - -impl DataclassFastSerializer { - pub fn new( - dict: *mut pyo3_ffi::PyObject, - opts: Opt, - default_calls: u8, - recursion: u8, - default: Option>, - ) -> Self { - DataclassFastSerializer { - dict: dict, - opts: opts, - default_calls: default_calls, - recursion: recursion, - default: default, - } - } -} - -impl Serialize for DataclassFastSerializer { - #[inline(never)] - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let len = unsafe { PyDict_GET_SIZE(self.dict) as usize }; - if unlikely!(len == 0) { - return serializer.serialize_map(Some(0)).unwrap().end(); - } - let mut map = serializer.serialize_map(None).unwrap(); - let mut pos = 0isize; - let mut key: *mut pyo3_ffi::PyObject = std::ptr::null_mut(); - let mut value: *mut pyo3_ffi::PyObject = std::ptr::null_mut(); - for _ in 0..=len.saturating_sub(1) { - unsafe { - pyo3_ffi::_PyDict_Next( - self.dict, - addr_of_mut!(pos), - addr_of_mut!(key), - addr_of_mut!(value), - std::ptr::null_mut(), - ) - }; - let pyvalue = PyObjectSerializer::new( - value, - self.opts, - self.default_calls, - self.recursion + 1, - self.default, - ); - if unlikely!(unsafe { ob_type!(key) != STR_TYPE }) { - err!(SerializeError::KeyMustBeStr) - } - let data = unicode_to_str(key); - if unlikely!(data.is_none()) { - err!(SerializeError::InvalidStr) - } - let key_as_str = data.unwrap(); - if unlikely!(key_as_str.as_bytes()[0] == b'_') { - continue; - } - map.serialize_key(key_as_str).unwrap(); - map.serialize_value(&pyvalue)?; - } - map.end() - } -} - -pub struct DataclassFallbackSerializer { - ptr: *mut pyo3_ffi::PyObject, - opts: Opt, - default_calls: u8, - recursion: u8, - default: Option>, -} - -impl DataclassFallbackSerializer { - pub fn new( - ptr: *mut pyo3_ffi::PyObject, - opts: Opt, - default_calls: u8, - recursion: u8, - default: Option>, - ) -> Self { - DataclassFallbackSerializer { - ptr: ptr, - opts: opts, - default_calls: default_calls, - recursion: recursion, - default: default, - } - } -} - -impl Serialize for DataclassFallbackSerializer { - #[inline(never)] - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let fields = ffi!(PyObject_GetAttr(self.ptr, DATACLASS_FIELDS_STR)); - ffi!(Py_DECREF(fields)); - let len = unsafe { PyDict_GET_SIZE(fields) as usize }; - if unlikely!(len == 0) { - return serializer.serialize_map(Some(0)).unwrap().end(); - } - let mut map = serializer.serialize_map(None).unwrap(); - let mut pos = 0isize; - let mut attr: *mut pyo3_ffi::PyObject = std::ptr::null_mut(); - let mut field: *mut pyo3_ffi::PyObject = std::ptr::null_mut(); - for _ in 0..=len - 1 { - unsafe { - pyo3_ffi::_PyDict_Next( - fields, - addr_of_mut!(pos), - addr_of_mut!(attr), - addr_of_mut!(field), - std::ptr::null_mut(), - ) - }; - let field_type = ffi!(PyObject_GetAttr(field, FIELD_TYPE_STR)); - ffi!(Py_DECREF(field_type)); - if unsafe { field_type != FIELD_TYPE.as_ptr() } { - continue; - } - let data = unicode_to_str(attr); - if unlikely!(data.is_none()) { - err!(SerializeError::InvalidStr); - } - let key_as_str = data.unwrap(); - if key_as_str.as_bytes()[0] == b'_' { - continue; - } - - let value = ffi!(PyObject_GetAttr(self.ptr, attr)); - ffi!(Py_DECREF(value)); - - map.serialize_key(key_as_str).unwrap(); - map.serialize_value(&PyObjectSerializer::new( - value, - self.opts, - self.default_calls, - self.recursion + 1, - self.default, - ))? - } - map.end() - } -} diff --git a/src/serialize/datetime.rs b/src/serialize/datetime.rs index 052de1ab..028d775d 100644 --- a/src/serialize/datetime.rs +++ b/src/serialize/datetime.rs @@ -1,244 +1,143 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -use crate::opt::*; -use crate::serialize::datetimelike::{DateTimeBuffer, DateTimeError, DateTimeLike, Offset}; -use crate::serialize::error::*; -use crate::typeref::*; -use serde::ser::{Serialize, Serializer}; - -macro_rules! write_double_digit { - ($buf:ident, $value:ident) => { - if $value < 10 { - $buf.push(b'0'); - } - $buf.extend_from_slice(itoa::Buffer::new().format($value).as_bytes()); - }; -} - -macro_rules! write_microsecond { - ($buf:ident, $microsecond:ident) => { - if $microsecond != 0 { - let mut buf = itoa::Buffer::new(); - let formatted = buf.format($microsecond); - $buf.extend_from_slice( - &[b'.', b'0', b'0', b'0', b'0', b'0', b'0'][..(7 - formatted.len())], - ); - $buf.extend_from_slice(formatted.as_bytes()); - } - }; -} - -#[repr(transparent)] -pub struct Date { - ptr: *mut pyo3_ffi::PyObject, -} - -impl Date { - pub fn new(ptr: *mut pyo3_ffi::PyObject) -> Self { - Date { ptr: ptr } +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2025-2026) + +use crate::ffi::{PyDateRef, PyDateTimeRef, PyTimeRef}; +use crate::opt::{NAIVE_UTC, OMIT_MICROSECONDS, Opt, UTC_Z}; +use crate::serialize::{ + error::SerializeError, + writer::{WriteExt, write_integer_u32}, +}; + +#[cold] +#[inline(never)] +pub(crate) fn write_time(ob: PyTimeRef, opts: Opt, buf: &mut B) -> Result<(), SerializeError> +where + B: ?Sized + WriteExt + bytes::BufMut, +{ + if ob.has_tz() { + return Err(SerializeError::TimeHasTzinfo); } - pub fn write_buf(&self, buf: &mut DateTimeBuffer) { - { - let year = ffi!(PyDateTime_GET_YEAR(self.ptr)) as i32; - let mut yearbuf = itoa::Buffer::new(); - let formatted = yearbuf.format(year); - if unlikely!(year < 1000) { - // date-fullyear = 4DIGIT - buf.extend_from_slice(&[b'0', b'0', b'0', b'0'][..(4 - formatted.len())]); - } - buf.extend_from_slice(formatted.as_bytes()); - } - buf.push(b'-'); - { - let month = ffi!(PyDateTime_GET_MONTH(self.ptr)) as u32; - write_double_digit!(buf, month); - } - buf.push(b'-'); - { - let day = ffi!(PyDateTime_GET_DAY(self.ptr)) as u32; - write_double_digit!(buf, day); - } + write_double_digit!(buf, ob.hour() as u32); + buf.put_u8(b':'); + write_double_digit!(buf, ob.minute() as u32); + buf.put_u8(b':'); + write_double_digit!(buf, ob.second() as u32); + if opt_disabled!(opts, OMIT_MICROSECONDS) { + write_microsecond!(buf, ob.microsecond()); } -} -impl Serialize for Date { - #[inline(never)] - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut buf = DateTimeBuffer::new(); - self.write_buf(&mut buf); - serializer.serialize_str(str_from_slice!(buf.as_ptr(), buf.len())) - } -} - -pub enum TimeError { - HasTimezone, -} - -pub struct Time { - ptr: *mut pyo3_ffi::PyObject, - opts: Opt, + Ok(()) } -impl Time { - pub fn new(ptr: *mut pyo3_ffi::PyObject, opts: Opt) -> Result { - if unsafe { (*(ptr as *mut pyo3_ffi::PyDateTime_Time)).hastzinfo == 1 } { - return Err(TimeError::HasTimezone); - } - Ok(Time { - ptr: ptr, - opts: opts, - }) - } - pub fn write_buf(&self, buf: &mut DateTimeBuffer) { - let hour = ffi!(PyDateTime_TIME_GET_HOUR(self.ptr)) as u8; - write_double_digit!(buf, hour); - buf.push(b':'); - let minute = ffi!(PyDateTime_TIME_GET_MINUTE(self.ptr)) as u8; - write_double_digit!(buf, minute); - buf.push(b':'); - let second = ffi!(PyDateTime_TIME_GET_SECOND(self.ptr)) as u8; - write_double_digit!(buf, second); - if self.opts & OMIT_MICROSECONDS == 0 { - let microsecond = ffi!(PyDateTime_TIME_GET_MICROSECOND(self.ptr)) as u32; - write_microsecond!(buf, microsecond); +#[cold] +#[inline(never)] +pub(crate) fn write_date(ob: PyDateRef, buf: &mut B) +where + B: ?Sized + WriteExt + bytes::BufMut, +{ + unsafe { + let year = ob.year(); + if year < 1000 { + cold_path!(); + // date-fullyear = 4DIGIT + buf.put_u8(b'0'); + if year < 100 { + buf.put_u8(b'0'); + } + if year < 10 { + buf.put_u8(b'0'); + } } + write_integer_u32(buf, year); + buf.put_u8(b'-'); + write_double_digit!(buf, ob.month()); + buf.put_u8(b'-'); + write_double_digit!(buf, ob.day()); } } -impl Serialize for Time { - #[inline(never)] - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut buf = DateTimeBuffer::new(); - self.write_buf(&mut buf); - serializer.serialize_str(str_from_slice!(buf.as_ptr(), buf.len())) - } -} - -pub struct DateTime { - ptr: *mut pyo3_ffi::PyObject, +#[inline(never)] +pub(crate) fn write_datetime( + ob: PyDateTimeRef, opts: Opt, -} - -impl DateTime { - pub fn new(ptr: *mut pyo3_ffi::PyObject, opts: Opt) -> Self { - DateTime { - ptr: ptr, - opts: opts, - } - } -} - -macro_rules! pydatetime_get { - ($fn: ident, $pyfn: ident, $ty: ident) => { - fn $fn(&self) -> $ty { - ffi!($pyfn(self.ptr)) as $ty - } - }; -} - -impl DateTimeLike for DateTime { - pydatetime_get!(year, PyDateTime_GET_YEAR, i32); - pydatetime_get!(month, PyDateTime_GET_MONTH, u8); - pydatetime_get!(day, PyDateTime_GET_DAY, u8); - pydatetime_get!(hour, PyDateTime_DATE_GET_HOUR, u8); - pydatetime_get!(minute, PyDateTime_DATE_GET_MINUTE, u8); - pydatetime_get!(second, PyDateTime_DATE_GET_SECOND, u8); - pydatetime_get!(microsecond, PyDateTime_DATE_GET_MICROSECOND, u32); - - fn millisecond(&self) -> u32 { - self.microsecond() / 1_000 - } - - fn nanosecond(&self) -> u32 { - self.microsecond() * 1_000 - } - - fn has_tz(&self) -> bool { - unsafe { (*(self.ptr as *mut pyo3_ffi::PyDateTime_DateTime)).hastzinfo == 1 } - } - - fn slow_offset(&self) -> Result { - let tzinfo = ffi!(PyDateTime_DATE_GET_TZINFO(self.ptr)); - if ffi!(PyObject_HasAttr(tzinfo, CONVERT_METHOD_STR)) == 1 { - // pendulum - let py_offset = call_method!(self.ptr, UTCOFFSET_METHOD_STR); - let offset = Offset { - second: ffi!(PyDateTime_DELTA_GET_SECONDS(py_offset)) as i32, - day: ffi!(PyDateTime_DELTA_GET_DAYS(py_offset)), - }; - ffi!(Py_DECREF(py_offset)); - Ok(offset) - } else if ffi!(PyObject_HasAttr(tzinfo, NORMALIZE_METHOD_STR)) == 1 { - // pytz - let method_ptr = call_method!(tzinfo, NORMALIZE_METHOD_STR, self.ptr); - let py_offset = call_method!(method_ptr, UTCOFFSET_METHOD_STR); - ffi!(Py_DECREF(method_ptr)); - let offset = Offset { - second: ffi!(PyDateTime_DELTA_GET_SECONDS(py_offset)) as i32, - day: ffi!(PyDateTime_DELTA_GET_DAYS(py_offset)), - }; - ffi!(Py_DECREF(py_offset)); - Ok(offset) - } else if ffi!(PyObject_HasAttr(tzinfo, DST_STR)) == 1 { - // dateutil/arrow, datetime.timezone.utc - let py_offset = call_method!(tzinfo, UTCOFFSET_METHOD_STR, self.ptr); - let offset = Offset { - second: ffi!(PyDateTime_DELTA_GET_SECONDS(py_offset)) as i32, - day: ffi!(PyDateTime_DELTA_GET_DAYS(py_offset)), - }; - ffi!(Py_DECREF(py_offset)); - Ok(offset) - } else { - Err(DateTimeError::LibraryUnsupported) - } - } - - #[cfg(Py_3_9)] - fn offset(&self) -> Result { - if !self.has_tz() { - Ok(Offset::default()) - } else { - let tzinfo = ffi!(PyDateTime_DATE_GET_TZINFO(self.ptr)); - if unsafe { ob_type!(tzinfo) == ZONEINFO_TYPE } { - // zoneinfo - let py_offset = call_method!(tzinfo, UTCOFFSET_METHOD_STR, self.ptr); - let offset = Offset { - second: ffi!(PyDateTime_DELTA_GET_SECONDS(py_offset)) as i32, - day: ffi!(PyDateTime_DELTA_GET_DAYS(py_offset)), - }; - ffi!(Py_DECREF(py_offset)); - Ok(offset) - } else { - self.slow_offset() + buf: &mut B, +) -> Result<(), SerializeError> +where + B: ?Sized + WriteExt + bytes::BufMut, +{ + { + let year = ob.year(); + debug_assert!(year >= 0); + if year < 1000 { + cold_path!(); + // date-fullyear = 4DIGIT + buf.put_u8(b'0'); + if year < 100 { + buf.put_u8(b'0'); + } + if year < 10 { + buf.put_u8(b'0'); } } + write_integer_u32(buf, year.cast_unsigned()); } - - #[cfg(not(Py_3_9))] - fn offset(&self) -> Result { - if !self.has_tz() { - Ok(Offset::default()) - } else { - self.slow_offset() + buf.put_u8(b'-'); + write_double_digit!(buf, ob.month() as u32); + buf.put_u8(b'-'); + write_double_digit!(buf, ob.day() as u32); + buf.put_u8(b'T'); + write_double_digit!(buf, ob.hour() as u32); + buf.put_u8(b':'); + write_double_digit!(buf, ob.minute() as u32); + buf.put_u8(b':'); + write_double_digit!(buf, ob.second() as u32); + if opt_disabled!(opts, OMIT_MICROSECONDS) { + let microsecond = ob.microsecond(); + if microsecond != 0 { + buf.put_u8(b'.'); + write_triple_digit!(buf, microsecond / 1_000); + write_triple_digit!(buf, microsecond % 1_000); } } -} - -impl Serialize for DateTime { - #[inline(never)] - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut buf = DateTimeBuffer::new(); - if self.write_buf(&mut buf, self.opts).is_err() { - err!(SerializeError::DatetimeLibraryUnsupported) + if ob.has_tz() || opt_enabled!(opts, NAIVE_UTC) { + match ob.offset() { + Some(offset) => { + let mut offset_second = offset.second; + if offset_second == 0 { + if opt_enabled!(opts, UTC_Z) { + buf.put_u8(b'Z'); + } else { + buf.put_slice(b"+00:00"); + } + } else { + if offset.day == -1 { + // datetime.timedelta(days=-1, seconds=68400) -> -05:00 + buf.put_u8(b'-'); + offset_second = 86400 - offset_second; + } else { + // datetime.timedelta(seconds=37800) -> +10:30 + buf.put_u8(b'+'); + } + let offset_minute = offset_second / 60; + let offset_hour = offset_minute / 60; + write_double_digit!(buf, offset_hour.cast_unsigned()); + buf.put_u8(b':'); + let mut offset_minute_print = offset_minute % 60; + // https://tools.ietf.org/html/rfc3339#section-5.8 + // "exactly 19 minutes and 32.13 seconds ahead of UTC" + // "closest representable UTC offset" + // "+20:00" + let offset_excess_second = + offset_second - (offset_minute_print * 60 + offset_hour * 3600); + if offset_excess_second >= 30 { + offset_minute_print += 1; + } + write_double_digit!(buf, offset_minute_print.cast_unsigned()); + } + } + None => { + return Err(SerializeError::DatetimeLibraryUnsupported); + } } - serializer.serialize_str(str_from_slice!(buf.as_ptr(), buf.len())) } + Ok(()) } diff --git a/src/serialize/datetimelike.rs b/src/serialize/datetimelike.rs deleted file mode 100644 index 95095cc5..00000000 --- a/src/serialize/datetimelike.rs +++ /dev/null @@ -1,174 +0,0 @@ -use crate::opt::*; - -#[derive(Debug)] -pub enum DateTimeError { - LibraryUnsupported, -} - -#[repr(transparent)] -pub struct DateTimeBuffer { - buf: arrayvec::ArrayVec, -} - -impl DateTimeBuffer { - pub fn new() -> DateTimeBuffer { - DateTimeBuffer { - buf: arrayvec::ArrayVec::::new(), - } - } - pub fn push(&mut self, value: u8) { - self.buf.push(value); - } - - pub fn extend_from_slice(&mut self, slice: &[u8]) { - self.buf.try_extend_from_slice(slice).unwrap(); - } - - pub fn as_ptr(&self) -> *const u8 { - self.buf.as_ptr() - } - - pub fn len(&self) -> usize { - self.buf.len() - } -} - -macro_rules! write_double_digit { - ($buf:ident, $value:expr) => { - if $value < 10 { - $buf.push(b'0'); - } - $buf.extend_from_slice(itoa::Buffer::new().format($value).as_bytes()); - }; -} - -macro_rules! write_triple_digit { - ($buf:ident, $value:expr) => { - if $value < 100 { - $buf.push(b'0'); - } - if $value < 10 { - $buf.push(b'0'); - } - $buf.extend_from_slice(itoa::Buffer::new().format($value).as_bytes()); - }; -} - -#[derive(Default)] -pub struct Offset { - pub day: i32, - pub second: i32, -} - -/// Trait providing a method to write a datetime-like object to a buffer in an RFC3339-compatible format. -/// -/// The provided `write_buf` method does not allocate, and is faster -/// than writing to a heap-allocated string. -pub trait DateTimeLike { - /// Returns the year component of the datetime. - fn year(&self) -> i32; - /// Returns the month component of the datetime. - fn month(&self) -> u8; - /// Returns the day component of the datetime. - fn day(&self) -> u8; - /// Returns the hour component of the datetime. - fn hour(&self) -> u8; - /// Returns the minute component of the datetime. - fn minute(&self) -> u8; - /// Returns the second component of the datetime. - fn second(&self) -> u8; - /// Returns the number of milliseconds since the whole non-leap second. - fn millisecond(&self) -> u32; - /// Returns the number of microseconds since the whole non-leap second. - fn microsecond(&self) -> u32; - /// Returns the number of nanoseconds since the whole non-leap second. - fn nanosecond(&self) -> u32; - - /// Is the object time-zone aware? - fn has_tz(&self) -> bool; - - //// python3.8 or below implementation of offset() - fn slow_offset(&self) -> Result; - - /// The offset of the timezone. - fn offset(&self) -> Result; - - /// Write `self` to a buffer in RFC3339 format, using `opts` to - /// customise if desired. - fn write_buf(&self, buf: &mut DateTimeBuffer, opts: Opt) -> Result<(), DateTimeError> { - { - let year = self.year(); - let mut yearbuf = itoa::Buffer::new(); - let formatted = yearbuf.format(year); - if unlikely!(year < 1000) { - // date-fullyear = 4DIGIT - buf.extend_from_slice(&[b'0', b'0', b'0', b'0'][..(4 - formatted.len())]); - } - buf.extend_from_slice(formatted.as_bytes()); - } - buf.push(b'-'); - write_double_digit!(buf, self.month()); - buf.push(b'-'); - write_double_digit!(buf, self.day()); - buf.push(b'T'); - write_double_digit!(buf, self.hour()); - buf.push(b':'); - write_double_digit!(buf, self.minute()); - buf.push(b':'); - write_double_digit!(buf, self.second()); - if opts & OMIT_MICROSECONDS == 0 { - let microsecond = self.microsecond(); - if microsecond != 0 { - buf.push(b'.'); - write_triple_digit!(buf, microsecond / 1_000); - write_triple_digit!(buf, microsecond % 1_000); - // Don't support writing nanoseconds for now. - // If requested, something like the following should work, - // and the `DateTimeBuffer` type alias should be changed to - // have length 35. - // let nanosecond = self.nanosecond(); - // if nanosecond % 1_000 != 0 { - // write_triple_digit!(buf, nanosecond % 1_000); - // } - } - } - if self.has_tz() || opts & NAIVE_UTC != 0 { - let offset = self.offset()?; - let mut offset_second = offset.second; - if offset_second == 0 { - if opts & UTC_Z != 0 { - buf.push(b'Z'); - } else { - buf.extend_from_slice(&[b'+', b'0', b'0', b':', b'0', b'0']); - } - } else { - // This branch is only really hit by the Python datetime implementation, - // since numpy datetimes are all converted to UTC. - if offset.day == -1 { - // datetime.timedelta(days=-1, seconds=68400) -> -05:00 - buf.push(b'-'); - offset_second = 86400 - offset_second; - } else { - // datetime.timedelta(seconds=37800) -> +10:30 - buf.push(b'+'); - } - let offset_minute = offset_second / 60; - let offset_hour = offset_minute / 60; - write_double_digit!(buf, offset_hour); - buf.push(b':'); - let mut offset_minute_print = offset_minute % 60; - // https://tools.ietf.org/html/rfc3339#section-5.8 - // "exactly 19 minutes and 32.13 seconds ahead of UTC" - // "closest representable UTC offset" - // "+20:00" - let offset_excess_second = - offset_second - (offset_minute_print * 60 + offset_hour * 3600); - if offset_excess_second >= 30 { - offset_minute_print += 1; - } - write_double_digit!(buf, offset_minute_print); - } - } - Ok(()) - } -} diff --git a/src/serialize/default.rs b/src/serialize/default.rs deleted file mode 100644 index ca1026ed..00000000 --- a/src/serialize/default.rs +++ /dev/null @@ -1,71 +0,0 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -use crate::opt::*; -use crate::serialize::error::*; -use crate::serialize::serializer::*; - -use serde::ser::{Serialize, Serializer}; - -use std::ptr::NonNull; - -pub struct DefaultSerializer { - ptr: *mut pyo3_ffi::PyObject, - opts: Opt, - default_calls: u8, - recursion: u8, - default: Option>, -} - -impl DefaultSerializer { - pub fn new( - ptr: *mut pyo3_ffi::PyObject, - opts: Opt, - default_calls: u8, - recursion: u8, - default: Option>, - ) -> Self { - DefaultSerializer { - ptr: ptr, - opts: opts, - default_calls: default_calls, - recursion: recursion, - default: default, - } - } -} - -impl Serialize for DefaultSerializer { - #[inline(never)] - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - match self.default { - Some(callable) => { - if unlikely!(self.default_calls == RECURSION_LIMIT) { - err!(SerializeError::DefaultRecursionLimit) - } - let default_obj = ffi!(PyObject_CallFunctionObjArgs( - callable.as_ptr(), - self.ptr, - std::ptr::null_mut() as *mut pyo3_ffi::PyObject - )); - if unlikely!(default_obj.is_null()) { - err!(SerializeError::UnsupportedType(nonnull!(self.ptr))) - } else { - let res = PyObjectSerializer::new( - default_obj, - self.opts, - self.default_calls + 1, - self.recursion, - self.default, - ) - .serialize(serializer); - ffi!(Py_DECREF(default_obj)); - res - } - } - None => err!(SerializeError::UnsupportedType(nonnull!(self.ptr))), - } - } -} diff --git a/src/serialize/dict.rs b/src/serialize/dict.rs deleted file mode 100644 index 00b1d285..00000000 --- a/src/serialize/dict.rs +++ /dev/null @@ -1,343 +0,0 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -use crate::ffi::PyDict_GET_SIZE; -use crate::opt::*; -use crate::serialize::datetime::*; -use crate::serialize::datetimelike::*; -use crate::serialize::error::*; -use crate::serialize::serializer::pyobject_to_obtype; -use crate::serialize::serializer::*; -use crate::serialize::uuid::*; -use crate::typeref::*; -use crate::unicode::*; -use inlinable_string::InlinableString; -use serde::ser::{Serialize, SerializeMap, Serializer}; -use smallvec::SmallVec; -use std::ptr::addr_of_mut; -use std::ptr::NonNull; - -pub struct Dict { - ptr: *mut pyo3_ffi::PyObject, - opts: Opt, - default_calls: u8, - recursion: u8, - default: Option>, -} - -impl Dict { - pub fn new( - ptr: *mut pyo3_ffi::PyObject, - opts: Opt, - default_calls: u8, - recursion: u8, - default: Option>, - ) -> Self { - Dict { - ptr: ptr, - opts: opts, - default_calls: default_calls, - recursion: recursion, - default: default, - } - } -} - -impl Serialize for Dict { - #[inline(never)] - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut map = serializer.serialize_map(None).unwrap(); - let mut pos = 0isize; - let mut key: *mut pyo3_ffi::PyObject = std::ptr::null_mut(); - let mut value: *mut pyo3_ffi::PyObject = std::ptr::null_mut(); - for _ in 0..=unsafe { PyDict_GET_SIZE(self.ptr) as usize } - 1 { - unsafe { - pyo3_ffi::_PyDict_Next( - self.ptr, - addr_of_mut!(pos), - addr_of_mut!(key), - addr_of_mut!(value), - std::ptr::null_mut(), - ) - }; - if unlikely!(unsafe { ob_type!(key) != STR_TYPE }) { - err!(SerializeError::KeyMustBeStr) - } - let key_as_str = unicode_to_str(key); - if unlikely!(key_as_str.is_none()) { - err!(SerializeError::InvalidStr) - } - let pyvalue = PyObjectSerializer::new( - value, - self.opts, - self.default_calls, - self.recursion + 1, - self.default, - ); - map.serialize_key(key_as_str.unwrap()).unwrap(); - map.serialize_value(&pyvalue)?; - } - map.end() - } -} - -pub struct DictSortedKey { - ptr: *mut pyo3_ffi::PyObject, - opts: Opt, - default_calls: u8, - recursion: u8, - default: Option>, -} - -impl DictSortedKey { - pub fn new( - ptr: *mut pyo3_ffi::PyObject, - opts: Opt, - default_calls: u8, - recursion: u8, - default: Option>, - ) -> Self { - DictSortedKey { - ptr: ptr, - opts: opts, - default_calls: default_calls, - recursion: recursion, - default: default, - } - } -} - -impl Serialize for DictSortedKey { - #[inline(never)] - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let len = unsafe { PyDict_GET_SIZE(self.ptr) as usize }; - let mut items: SmallVec<[(&str, *mut pyo3_ffi::PyObject); 8]> = - SmallVec::with_capacity(len); - let mut pos = 0isize; - let mut key: *mut pyo3_ffi::PyObject = std::ptr::null_mut(); - let mut value: *mut pyo3_ffi::PyObject = std::ptr::null_mut(); - for _ in 0..=len - 1 { - unsafe { - pyo3_ffi::_PyDict_Next( - self.ptr, - addr_of_mut!(pos), - addr_of_mut!(key), - addr_of_mut!(value), - std::ptr::null_mut(), - ) - }; - if unlikely!(unsafe { ob_type!(key) != STR_TYPE }) { - err!(SerializeError::KeyMustBeStr) - } - let data = unicode_to_str(key); - if unlikely!(data.is_none()) { - err!(SerializeError::InvalidStr) - } - items.push((data.unwrap(), value)); - } - - items.sort_unstable_by(|a, b| a.0.cmp(b.0)); - - let mut map = serializer.serialize_map(None).unwrap(); - for (key, val) in items.iter() { - map.serialize_entry( - key, - &PyObjectSerializer::new( - *val, - self.opts, - self.default_calls, - self.recursion + 1, - self.default, - ), - )?; - } - map.end() - } -} - -pub struct DictNonStrKey { - ptr: *mut pyo3_ffi::PyObject, - opts: Opt, - default_calls: u8, - recursion: u8, - default: Option>, -} - -impl DictNonStrKey { - pub fn new( - ptr: *mut pyo3_ffi::PyObject, - opts: Opt, - default_calls: u8, - recursion: u8, - default: Option>, - ) -> Self { - DictNonStrKey { - ptr: ptr, - opts: opts, - default_calls: default_calls, - recursion: recursion, - default: default, - } - } - - fn pyobject_to_string( - &self, - key: *mut pyo3_ffi::PyObject, - opts: crate::opt::Opt, - ) -> Result { - match pyobject_to_obtype(key, opts) { - ObType::None => Ok(InlinableString::from("null")), - ObType::Bool => { - let key_as_str = if unsafe { key == TRUE } { - "true" - } else { - "false" - }; - Ok(InlinableString::from(key_as_str)) - } - ObType::Int => { - let ival = ffi!(PyLong_AsLongLong(key)); - if unlikely!(ival == -1 && !ffi!(PyErr_Occurred()).is_null()) { - ffi!(PyErr_Clear()); - let uval = ffi!(PyLong_AsUnsignedLongLong(key)); - if unlikely!(uval == u64::MAX && !ffi!(PyErr_Occurred()).is_null()) { - return Err(SerializeError::DictIntegerKey64Bit); - } - Ok(InlinableString::from(itoa::Buffer::new().format(uval))) - } else { - Ok(InlinableString::from(itoa::Buffer::new().format(ival))) - } - } - ObType::Float => { - let val = ffi!(PyFloat_AS_DOUBLE(key)); - if !val.is_finite() { - Ok(InlinableString::from("null")) - } else { - Ok(InlinableString::from(ryu::Buffer::new().format_finite(val))) - } - } - ObType::Datetime => { - let mut buf = DateTimeBuffer::new(); - let dt = DateTime::new(key, opts); - if dt.write_buf(&mut buf, opts).is_err() { - return Err(SerializeError::DatetimeLibraryUnsupported); - } - let key_as_str = str_from_slice!(buf.as_ptr(), buf.len()); - Ok(InlinableString::from(key_as_str)) - } - ObType::Date => { - let mut buf = DateTimeBuffer::new(); - Date::new(key).write_buf(&mut buf); - let key_as_str = str_from_slice!(buf.as_ptr(), buf.len()); - Ok(InlinableString::from(key_as_str)) - } - ObType::Time => match Time::new(key, opts) { - Ok(val) => { - let mut buf = DateTimeBuffer::new(); - val.write_buf(&mut buf); - let key_as_str = str_from_slice!(buf.as_ptr(), buf.len()); - Ok(InlinableString::from(key_as_str)) - } - Err(TimeError::HasTimezone) => Err(SerializeError::TimeHasTzinfo), - }, - ObType::Uuid => { - let mut buf = arrayvec::ArrayVec::::new(); - UUID::new(key).write_buf(&mut buf); - let key_as_str = str_from_slice!(buf.as_ptr(), buf.len()); - Ok(InlinableString::from(key_as_str)) - } - ObType::Enum => { - let value = ffi!(PyObject_GetAttr(key, VALUE_STR)); - ffi!(Py_DECREF(value)); - self.pyobject_to_string(value, opts) - } - ObType::Str => { - // because of ObType::Enum - let uni = unicode_to_str(key); - if unlikely!(uni.is_none()) { - Err(SerializeError::InvalidStr) - } else { - Ok(InlinableString::from(uni.unwrap())) - } - } - ObType::StrSubclass => { - let uni = unicode_to_str_via_ffi(key); - if unlikely!(uni.is_none()) { - Err(SerializeError::InvalidStr) - } else { - Ok(InlinableString::from(uni.unwrap())) - } - } - ObType::Tuple - | ObType::NumpyScalar - | ObType::NumpyArray - | ObType::Dict - | ObType::List - | ObType::Dataclass - | ObType::Unknown => Err(SerializeError::DictKeyInvalidType), - } - } -} - -impl Serialize for DictNonStrKey { - #[inline(never)] - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let len = unsafe { PyDict_GET_SIZE(self.ptr) as usize }; - let mut items: SmallVec<[(InlinableString, *mut pyo3_ffi::PyObject); 8]> = - SmallVec::with_capacity(len); - let mut pos = 0isize; - let mut key: *mut pyo3_ffi::PyObject = std::ptr::null_mut(); - let mut value: *mut pyo3_ffi::PyObject = std::ptr::null_mut(); - let opts = self.opts & NOT_PASSTHROUGH; - for _ in 0..=len - 1 { - unsafe { - pyo3_ffi::_PyDict_Next( - self.ptr, - addr_of_mut!(pos), - addr_of_mut!(key), - addr_of_mut!(value), - std::ptr::null_mut(), - ) - }; - if is_type!(ob_type!(key), STR_TYPE) { - let data = unicode_to_str(key); - if unlikely!(data.is_none()) { - err!(SerializeError::InvalidStr) - } - items.push((InlinableString::from(data.unwrap()), value)); - } else { - match self.pyobject_to_string(key, opts) { - Ok(key_as_str) => items.push((key_as_str, value)), - Err(err) => err!(err), - } - } - } - - if opts & SORT_KEYS != 0 { - items.sort_unstable_by(|a, b| a.0.cmp(&b.0)); - } - - let mut map = serializer.serialize_map(None).unwrap(); - for (key, val) in items.iter() { - map.serialize_entry( - str_from_slice!(key.as_ptr(), key.len()), - &PyObjectSerializer::new( - *val, - self.opts, - self.default_calls, - self.recursion + 1, - self.default, - ), - )?; - } - map.end() - } -} diff --git a/src/serialize/error.rs b/src/serialize/error.rs index 22954518..ee207bf3 100644 --- a/src/serialize/error.rs +++ b/src/serialize/error.rs @@ -1,15 +1,17 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2021-2026) -use crate::error::INVALID_STR; -use std::ffi::CStr; -use std::ptr::NonNull; +use crate::ffi::PyStrRef; +use core::ffi::CStr; +use core::ptr::NonNull; -pub enum SerializeError { +pub(crate) enum SerializeError { DatetimeLibraryUnsupported, DefaultRecursionLimit, Integer53Bits, Integer64Bits, InvalidStr, + InvalidFragment, KeyMustBeStr, RecursionLimit, TimeHasTzinfo, @@ -17,22 +19,30 @@ pub enum SerializeError { DictKeyInvalidType, NumpyMalformed, NumpyNotCContiguous, + NumpyNotNativeEndian, NumpyUnsupportedDatatype, - UnsupportedType(NonNull), + NumpyUnsupportedDatetimeUnit(PyStrRef), + UnsupportedType(NonNull), } -impl std::fmt::Display for SerializeError { +impl core::fmt::Display for SerializeError { #[cold] - #[cfg_attr(feature = "unstable-simd", optimize(size))] - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + #[cfg_attr(feature = "optimize", optimize(size))] + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { match *self { - SerializeError::DatetimeLibraryUnsupported => write!(f, "datetime's timezone library is not supported: use datetime.timezone.utc, pendulum, pytz, or dateutil"), + SerializeError::DatetimeLibraryUnsupported => write!( + f, + "datetime's timezone library is not supported: use datetime.timezone.utc, pendulum, pytz, or dateutil" + ), SerializeError::DefaultRecursionLimit => { write!(f, "default serializer exceeds recursion limit") } SerializeError::Integer53Bits => write!(f, "Integer exceeds 53-bit range"), SerializeError::Integer64Bits => write!(f, "Integer exceeds 64-bit range"), - SerializeError::InvalidStr => write!(f, "{}", INVALID_STR), + SerializeError::InvalidStr => write!(f, "{}", crate::util::INVALID_STR), + SerializeError::InvalidFragment => { + write!(f, "orjson.Fragment's content is not of type bytes or str") + } SerializeError::KeyMustBeStr => write!(f, "Dict key must be str"), SerializeError::RecursionLimit => write!(f, "Recursion limit reached"), SerializeError::TimeHasTzinfo => write!(f, "datetime.time must not have tzinfo set"), @@ -47,12 +57,21 @@ impl std::fmt::Display for SerializeError { f, "numpy array is not C contiguous; use ndarray.tolist() in default" ), + SerializeError::NumpyNotNativeEndian => { + write!(f, "numpy array is not native-endianness") + } SerializeError::NumpyUnsupportedDatatype => { write!(f, "unsupported datatype in numpy array") } + SerializeError::NumpyUnsupportedDatetimeUnit(msg) => { + write!(f, "{}", msg.as_str().unwrap()) + } SerializeError::UnsupportedType(ptr) => { - let name = unsafe { CStr::from_ptr((*ob_type!(ptr.as_ptr())).tp_name).to_string_lossy() }; - write!(f, "Type is not JSON serializable: {}", name) + let name = unsafe { + CStr::from_ptr((*crate::ffi::PyObject_Type(ptr.as_ptr())).tp_name) + .to_string_lossy() + }; + write!(f, "Type is not JSON serializable: {name}") } } } diff --git a/src/serialize/int.rs b/src/serialize/int.rs deleted file mode 100644 index 0e595abe..00000000 --- a/src/serialize/int.rs +++ /dev/null @@ -1,90 +0,0 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -use crate::serialize::error::*; -use serde::ser::{Serialize, Serializer}; - -// https://tools.ietf.org/html/rfc7159#section-6 -// "[-(2**53)+1, (2**53)-1]" -const STRICT_INT_MIN: i64 = -9007199254740991; -const STRICT_INT_MAX: i64 = 9007199254740991; - -#[repr(transparent)] -pub struct IntSerializer { - ptr: *mut pyo3_ffi::PyObject, -} - -impl IntSerializer { - pub fn new(ptr: *mut pyo3_ffi::PyObject) -> Self { - IntSerializer { ptr: ptr } - } -} - -impl Serialize for IntSerializer { - #[inline] - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let val = ffi!(PyLong_AsLongLong(self.ptr)); - if unlikely!(val == -1 && !ffi!(PyErr_Occurred()).is_null()) { - UIntSerializer::new(self.ptr).serialize(serializer) - } else { - serializer.serialize_i64(val) - } - } -} - -#[repr(transparent)] -pub struct UIntSerializer { - ptr: *mut pyo3_ffi::PyObject, -} - -impl UIntSerializer { - pub fn new(ptr: *mut pyo3_ffi::PyObject) -> Self { - UIntSerializer { ptr: ptr } - } -} - -impl Serialize for UIntSerializer { - #[cold] - #[inline(never)] - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - ffi!(PyErr_Clear()); - let val = ffi!(PyLong_AsUnsignedLongLong(self.ptr)); - if unlikely!(val == u64::MAX && !ffi!(PyErr_Occurred()).is_null()) { - err!(SerializeError::Integer64Bits) - } - serializer.serialize_u64(val) - } -} - -#[repr(transparent)] -pub struct Int53Serializer { - ptr: *mut pyo3_ffi::PyObject, -} - -impl Int53Serializer { - pub fn new(ptr: *mut pyo3_ffi::PyObject) -> Self { - Int53Serializer { ptr: ptr } - } -} - -impl Serialize for Int53Serializer { - #[cold] - #[inline(never)] - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let val = ffi!(PyLong_AsLongLong(self.ptr)); - if unlikely!(val == -1 && !ffi!(PyErr_Occurred()).is_null()) - || !(STRICT_INT_MIN..=STRICT_INT_MAX).contains(&val) - { - err!(SerializeError::Integer53Bits) - } - serializer.serialize_i64(val) - } -} diff --git a/src/serialize/list.rs b/src/serialize/list.rs deleted file mode 100644 index 498a4b3d..00000000 --- a/src/serialize/list.rs +++ /dev/null @@ -1,59 +0,0 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -use crate::opt::*; -use crate::serialize::serializer::*; - -use serde::ser::{Serialize, SerializeSeq, Serializer}; -use std::ptr::NonNull; - -pub struct ListSerializer { - ptr: *mut pyo3_ffi::PyObject, - opts: Opt, - default_calls: u8, - recursion: u8, - default: Option>, -} - -impl ListSerializer { - pub fn new( - ptr: *mut pyo3_ffi::PyObject, - opts: Opt, - default_calls: u8, - recursion: u8, - default: Option>, - ) -> Self { - ListSerializer { - ptr: ptr, - opts: opts, - default_calls: default_calls, - recursion: recursion, - default: default, - } - } -} - -impl Serialize for ListSerializer { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut seq = serializer.serialize_seq(None).unwrap(); - let slice: &[*mut pyo3_ffi::PyObject] = unsafe { - std::slice::from_raw_parts( - (*(self.ptr as *mut pyo3_ffi::PyListObject)).ob_item, - ffi!(PyList_GET_SIZE(self.ptr)) as usize, - ) - }; - for &each in slice { - let value = PyObjectSerializer::new( - each, - self.opts, - self.default_calls, - self.recursion + 1, - self.default, - ); - seq.serialize_element(&value)?; - } - seq.end() - } -} diff --git a/src/serialize/mod.rs b/src/serialize/mod.rs index 1e9efbb5..25ffb930 100644 --- a/src/serialize/mod.rs +++ b/src/serialize/mod.rs @@ -1,19 +1,15 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2021-2026) -mod dataclass; -mod datetime; -#[macro_use] -mod datetimelike; -mod default; -mod dict; +pub(crate) mod datetime; mod error; -mod int; -mod list; mod numpy; +mod obtype; +mod per_type; mod serializer; -mod str; -mod tuple; +mod state; mod uuid; -mod writer; +pub(crate) mod writer; -pub use serializer::serialize; +pub(crate) use serializer::serialize; +pub(crate) use writer::set_str_formatter_fn; diff --git a/src/serialize/non_str.rs b/src/serialize/non_str.rs new file mode 100644 index 00000000..aef782f5 --- /dev/null +++ b/src/serialize/non_str.rs @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2024-2026) + +use crate::ffi::{ + PyDateRef, PyDateTimeRef, PyFloatRef, PyStrRef, PyStrSubclassRef, PyTimeRef, PyUuidRef, +}; +use crate::serialize::{ + datetime::{write_date, write_datetime, write_time}, + error::SerializeError, + obtype::ObType, + writer::{ + SmallFixedBuffer, pyobject_to_obtype, write_float64, write_integer_i64, write_integer_u64, + }, +}; +use crate::typeref::{TRUE, VALUE_STR}; + +fn non_str_str(key: PyStrRef) -> Result { + // because of ObType::Enum + match key.as_str() { + Some(uni) => Ok(String::from(uni)), + None => { + cold_path!(); + Err(SerializeError::InvalidStr) + } + } +} + +fn non_str_str_subclass(key: PyStrSubclassRef) -> Result { + match key.as_str() { + Some(uni) => Ok(String::from(uni)), + None => { + cold_path!(); + Err(SerializeError::InvalidStr) + } + } +} + +#[allow(clippy::unnecessary_wraps)] +fn non_str_date(key: PyDateRef) -> Result { + let mut buf = SmallFixedBuffer::new(); + write_date(key, &mut buf); + Ok(buf.to_string()) +} + +fn non_str_datetime(key: PyDateTimeRef, opts: crate::opt::Opt) -> Result { + let mut buf = SmallFixedBuffer::new(); + if write_datetime(key, opts, &mut buf).is_err() { + return Err(SerializeError::DatetimeLibraryUnsupported); + } + Ok(buf.to_string()) +} + +fn non_str_time(key: PyTimeRef, opts: crate::opt::Opt) -> Result { + let mut buf = SmallFixedBuffer::new(); + write_time(key, opts, &mut buf)?; + Ok(buf.to_string()) +} + +#[allow(clippy::unnecessary_wraps)] +fn non_str_uuid(key: PyUuidRef) -> Result { + let mut buf = SmallFixedBuffer::new(); + UUID::new(key).write_buf(&mut buf); + Ok(buf.to_string()) +} + +#[allow(clippy::unnecessary_wraps)] +fn non_str_float(ob: PyFloatRef) -> Result { + let mut buf = SmallFixedBuffer::new(); + write_float64(&mut buf, ob.value()); + Ok(buf.to_string()) +} + +#[allow(clippy::unnecessary_wraps)] +fn non_str_int(key: *mut crate::ffi::PyObject) -> Result { + let ival = unsafe { crate::ffi::PyLong_AsLongLong(key) }; + if ival == -1 && unsafe { !crate::ffi::PyErr_Occurred().is_null() } { + cold_path!(); + unsafe { crate::ffi::PyErr_Clear() }; + let uval = unsafe { crate::ffi::PyLong_AsUnsignedLongLong(key) }; + if uval == u64::MAX && unsafe { !crate::ffi::PyErr_Occurred().is_null() } { + cold_path!(); + return Err(SerializeError::DictIntegerKey64Bit); + } + let mut buf = SmallFixedBuffer::new(); + write_integer_u64(&mut buf, uval); + Ok(buf.to_string()) + } else { + let mut buf = SmallFixedBuffer::new(); + write_integer_i64(&mut buf, ival); + Ok(buf.to_string()) + } +} + +#[inline(never)] +#[cfg_attr(feature = "optimize", optimize(size))] +pub(crate) fn pyobject_to_string( + key: *mut crate::ffi::PyObject, + opts: crate::opt::Opt, +) -> Result { + unsafe { + match pyobject_to_obtype(key, opts) { + ObType::None => Ok(String::from("null")), + ObType::Bool => { + if unsafe { core::ptr::eq(key, TRUE) } { + Ok(String::from("true")) + } else { + Ok(String::from("false")) + } + } + ObType::Int => non_str_int(key), + ObType::Float => non_str_float(PyFloatRef::from_ptr_unchecked(key)), + ObType::Datetime => non_str_datetime(PyDateTimeRef::from_ptr_unchecked(key), opts), + ObType::Date => non_str_date(PyDateRef::from_ptr_unchecked(key)), + ObType::Time => non_str_time(PyTimeRef::from_ptr_unchecked(key), opts), + ObType::Uuid => non_str_uuid(PyUuidRef::from_ptr_unchecked(key)), + ObType::Enum => { + let value = unsafe { crate::ffi::PyObject_GetAttr(key, VALUE_STR) }; + debug_assert!(unsafe { crate::ffi::Py_REFCNT(value) >= 2 }); + let ret = pyobject_to_string(value, opts); + unsafe { crate::ffi::Py_DECREF(value) }; + ret + } + ObType::Str => non_str_str(PyStrRef::from_ptr_unchecked(key)), + ObType::StrSubclass => non_str_str_subclass(PyStrSubclassRef::from_ptr_unchecked(key)), + ObType::Tuple + | ObType::NumpyScalar + | ObType::NumpyArray + | ObType::Dict + | ObType::List + | ObType::Dataclass + | ObType::Fragment + | ObType::Unknown => Err(SerializeError::DictKeyInvalidType), + } + } +} diff --git a/src/serialize/numpy.rs b/src/serialize/numpy.rs deleted file mode 100644 index 1968cb78..00000000 --- a/src/serialize/numpy.rs +++ /dev/null @@ -1,861 +0,0 @@ -use crate::opt::*; -use crate::serialize::datetimelike::{DateTimeBuffer, DateTimeError, DateTimeLike, Offset}; -use crate::typeref::{ARRAY_STRUCT_STR, DESCR_STR, DTYPE_STR, NUMPY_TYPES}; -use chrono::{Datelike, NaiveDate, NaiveDateTime, Timelike}; -use pyo3_ffi::*; -use serde::ser::{self, Serialize, SerializeSeq, Serializer}; -use std::convert::TryInto; -use std::fmt; -use std::ops::DerefMut; -use std::os::raw::{c_char, c_int, c_void}; - -macro_rules! slice { - ($ptr:expr, $size:expr) => { - unsafe { std::slice::from_raw_parts($ptr, $size) } - }; -} - -pub fn is_numpy_scalar(ob_type: *mut PyTypeObject) -> bool { - if unsafe { NUMPY_TYPES.is_none() } { - false - } else { - let scalar_types = unsafe { NUMPY_TYPES.as_ref().unwrap() }; - ob_type == scalar_types.float64 - || ob_type == scalar_types.float32 - || ob_type == scalar_types.int64 - || ob_type == scalar_types.int32 - || ob_type == scalar_types.int8 - || ob_type == scalar_types.uint64 - || ob_type == scalar_types.uint32 - || ob_type == scalar_types.uint8 - || ob_type == scalar_types.bool_ - || ob_type == scalar_types.datetime64 - } -} - -pub fn is_numpy_array(ob_type: *mut PyTypeObject) -> bool { - if unsafe { NUMPY_TYPES.is_none() } { - false - } else { - unsafe { ob_type == NUMPY_TYPES.as_ref().unwrap().array } - } -} - -#[repr(C)] -pub struct PyCapsule { - pub ob_refcnt: Py_ssize_t, - pub ob_type: *mut PyTypeObject, - pub pointer: *mut c_void, - pub name: *const c_char, - pub context: *mut c_void, - pub destructor: *mut c_void, // should be typedef void (*PyCapsule_Destructor)(PyObject *); -} - -// https://docs.scipy.org/doc/numpy/reference/arrays.interface.html#c.__array_struct__ - -#[repr(C)] -pub struct PyArrayInterface { - pub two: c_int, - pub nd: c_int, - pub typekind: c_char, - pub itemsize: c_int, - pub flags: c_int, - pub shape: *mut Py_intptr_t, - pub strides: *mut Py_intptr_t, - pub data: *mut c_void, - pub descr: *mut PyObject, -} - -#[derive(Clone, Copy)] -pub enum ItemType { - BOOL, - DATETIME64(NumpyDatetimeUnit), - F32, - F64, - I8, - I32, - I64, - U8, - U32, - U64, -} - -impl ItemType { - fn find(array: *mut PyArrayInterface, ptr: *mut PyObject) -> Option { - match unsafe { ((*array).typekind, (*array).itemsize) } { - (098, 1) => Some(ItemType::BOOL), - (077, 8) => { - let unit = NumpyDatetimeUnit::from_pyobject(ptr); - Some(ItemType::DATETIME64(unit)) - } - (102, 4) => Some(ItemType::F32), - (102, 8) => Some(ItemType::F64), - (105, 1) => Some(ItemType::I8), - (105, 4) => Some(ItemType::I32), - (105, 8) => Some(ItemType::I64), - (117, 1) => Some(ItemType::U8), - (117, 4) => Some(ItemType::U32), - (117, 8) => Some(ItemType::U64), - _ => None, - } - } -} -pub enum PyArrayError { - Malformed, - NotContiguous, - UnsupportedDataType, -} - -// >>> arr = numpy.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]], numpy.int32) -// >>> arr.ndim -// 3 -// >>> arr.shape -// (2, 2, 2) -// >>> arr.strides -// (16, 8, 4) -pub struct NumpyArray { - array: *mut PyArrayInterface, - position: Vec, - children: Vec, - depth: usize, - capsule: *mut PyCapsule, - kind: ItemType, - opts: Opt, -} - -impl NumpyArray { - #[inline(never)] - pub fn new(ptr: *mut PyObject, opts: Opt) -> Result { - let capsule = ffi!(PyObject_GetAttr(ptr, ARRAY_STRUCT_STR)); - let array = unsafe { (*(capsule as *mut PyCapsule)).pointer as *mut PyArrayInterface }; - if unsafe { (*array).two != 2 } { - ffi!(Py_DECREF(capsule)); - Err(PyArrayError::Malformed) - } else if unsafe { (*array).flags } & 0x1 != 0x1 { - ffi!(Py_DECREF(capsule)); - Err(PyArrayError::NotContiguous) - } else { - let num_dimensions = unsafe { (*array).nd as usize }; - if num_dimensions == 0 { - ffi!(Py_DECREF(capsule)); - return Err(PyArrayError::UnsupportedDataType); - } - match ItemType::find(array, ptr) { - None => Err(PyArrayError::UnsupportedDataType), - Some(kind) => { - let mut pyarray = NumpyArray { - array: array, - position: vec![0; num_dimensions], - children: Vec::with_capacity(num_dimensions), - depth: 0, - capsule: capsule as *mut PyCapsule, - kind: kind, - opts, - }; - if pyarray.dimensions() > 1 { - pyarray.build(); - } - Ok(pyarray) - } - } - } - } - - fn child_from_parent(&self, position: Vec, num_children: usize) -> Self { - let mut arr = NumpyArray { - array: self.array, - position: position, - children: Vec::with_capacity(num_children), - depth: self.depth + 1, - capsule: self.capsule, - kind: self.kind, - opts: self.opts, - }; - arr.build(); - arr - } - - fn build(&mut self) { - if self.depth < self.dimensions() - 1 { - for i in 0..=self.shape()[self.depth] - 1 { - let mut position: Vec = self.position.to_vec(); - position[self.depth] = i; - let num_children: usize = if self.depth < self.dimensions() - 2 { - self.shape()[self.depth + 1] as usize - } else { - 0 - }; - self.children - .push(self.child_from_parent(position, num_children)) - } - } - } - - fn data(&self) -> *const c_void { - let offset = self - .strides() - .iter() - .zip(self.position.iter().copied()) - .take(self.depth) - .map(|(a, b)| a * b) - .sum::(); - unsafe { (*self.array).data.offset(offset) } - } - - fn num_items(&self) -> usize { - self.shape()[self.shape().len() - 1] as usize - } - - fn dimensions(&self) -> usize { - unsafe { (*self.array).nd as usize } - } - - fn shape(&self) -> &[isize] { - slice!((*self.array).shape as *const isize, self.dimensions()) - } - - fn strides(&self) -> &[isize] { - slice!((*self.array).strides as *const isize, self.dimensions()) - } -} - -impl Drop for NumpyArray { - fn drop(&mut self) { - if self.depth == 0 { - ffi!(Py_DECREF(self.array as *mut pyo3_ffi::PyObject)); - ffi!(Py_DECREF(self.capsule as *mut pyo3_ffi::PyObject)); - } - } -} - -impl Serialize for NumpyArray { - #[inline(never)] - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut seq = serializer.serialize_seq(None).unwrap(); - - if self.depth >= self.shape().len() || self.shape()[self.depth] != 0 { - if !self.children.is_empty() { - for child in &self.children { - seq.serialize_element(child).unwrap(); - } - } else { - let data_ptr = self.data(); - let num_items = self.num_items(); - match self.kind { - ItemType::F64 => { - let slice: &[f64] = slice!(data_ptr as *const f64, num_items); - for &each in slice.iter() { - seq.serialize_element(&DataTypeF64 { obj: each }).unwrap(); - } - } - ItemType::F32 => { - let slice: &[f32] = slice!(data_ptr as *const f32, num_items); - for &each in slice.iter() { - seq.serialize_element(&DataTypeF32 { obj: each }).unwrap(); - } - } - ItemType::I64 => { - let slice: &[i64] = slice!(data_ptr as *const i64, num_items); - for &each in slice.iter() { - seq.serialize_element(&DataTypeI64 { obj: each }).unwrap(); - } - } - ItemType::I32 => { - let slice: &[i32] = slice!(data_ptr as *const i32, num_items); - for &each in slice.iter() { - seq.serialize_element(&DataTypeI32 { obj: each }).unwrap(); - } - } - ItemType::I8 => { - let slice: &[i8] = slice!(data_ptr as *const i8, num_items); - for &each in slice.iter() { - seq.serialize_element(&DataTypeI8 { obj: each }).unwrap(); - } - } - ItemType::U8 => { - let slice: &[u8] = slice!(data_ptr as *const u8, num_items); - for &each in slice.iter() { - seq.serialize_element(&DataTypeU8 { obj: each }).unwrap(); - } - } - ItemType::U32 => { - let slice: &[u32] = slice!(data_ptr as *const u32, num_items); - for &each in slice.iter() { - seq.serialize_element(&DataTypeU32 { obj: each }).unwrap(); - } - } - ItemType::U64 => { - let slice: &[u64] = slice!(data_ptr as *const u64, num_items); - for &each in slice.iter() { - seq.serialize_element(&DataTypeU64 { obj: each }).unwrap(); - } - } - ItemType::BOOL => { - let slice: &[u8] = slice!(data_ptr as *const u8, num_items); - for &each in slice.iter() { - seq.serialize_element(&DataTypeBOOL { obj: each }).unwrap(); - } - } - ItemType::DATETIME64(unit) => { - let slice: &[i64] = slice!(data_ptr as *const i64, num_items); - for &each in slice.iter() { - let dt = unit - .datetime(each, self.opts) - .map_err(NumpyDateTimeError::into_serde_err)?; - seq.serialize_element(&dt).unwrap(); - } - } - } - } - } - seq.end() - } -} - -#[repr(transparent)] -struct DataTypeF32 { - obj: f32, -} - -impl Serialize for DataTypeF32 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_f32(self.obj) - } -} - -#[repr(transparent)] -pub struct DataTypeF64 { - obj: f64, -} - -impl Serialize for DataTypeF64 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_f64(self.obj) - } -} - -#[repr(transparent)] -pub struct DataTypeI8 { - obj: i8, -} - -impl Serialize for DataTypeI8 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_i8(self.obj) - } -} - -#[repr(transparent)] -pub struct DataTypeI32 { - obj: i32, -} - -impl Serialize for DataTypeI32 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_i32(self.obj) - } -} - -#[repr(transparent)] -pub struct DataTypeI64 { - obj: i64, -} - -impl Serialize for DataTypeI64 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_i64(self.obj) - } -} - -#[repr(transparent)] -pub struct DataTypeU8 { - obj: u8, -} - -impl Serialize for DataTypeU8 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_u8(self.obj) - } -} - -#[repr(transparent)] -pub struct DataTypeU32 { - obj: u32, -} - -impl Serialize for DataTypeU32 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_u32(self.obj) - } -} - -#[repr(transparent)] -pub struct DataTypeU64 { - obj: u64, -} - -impl Serialize for DataTypeU64 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_u64(self.obj) - } -} - -#[repr(transparent)] -pub struct DataTypeBOOL { - obj: u8, -} - -impl Serialize for DataTypeBOOL { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_bool(self.obj == 1) - } -} - -pub struct NumpyScalar { - ptr: *mut pyo3_ffi::PyObject, - opts: Opt, -} - -impl NumpyScalar { - pub fn new(ptr: *mut PyObject, opts: Opt) -> Self { - NumpyScalar { ptr, opts } - } -} - -impl Serialize for NumpyScalar { - #[inline(never)] - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - unsafe { - let ob_type = ob_type!(self.ptr); - let scalar_types = NUMPY_TYPES.deref_mut().as_ref().unwrap(); - if ob_type == scalar_types.float64 { - (*(self.ptr as *mut NumpyFloat64)).serialize(serializer) - } else if ob_type == scalar_types.float32 { - (*(self.ptr as *mut NumpyFloat32)).serialize(serializer) - } else if ob_type == scalar_types.int64 { - (*(self.ptr as *mut NumpyInt64)).serialize(serializer) - } else if ob_type == scalar_types.int32 { - (*(self.ptr as *mut NumpyInt32)).serialize(serializer) - } else if ob_type == scalar_types.int8 { - (*(self.ptr as *mut NumpyInt8)).serialize(serializer) - } else if ob_type == scalar_types.uint64 { - (*(self.ptr as *mut NumpyUint64)).serialize(serializer) - } else if ob_type == scalar_types.uint32 { - (*(self.ptr as *mut NumpyUint32)).serialize(serializer) - } else if ob_type == scalar_types.uint8 { - (*(self.ptr as *mut NumpyUint8)).serialize(serializer) - } else if ob_type == scalar_types.bool_ { - (*(self.ptr as *mut NumpyBool)).serialize(serializer) - } else if ob_type == scalar_types.datetime64 { - let unit = NumpyDatetimeUnit::from_pyobject(self.ptr); - let obj = &*(self.ptr as *mut NumpyDatetime64); - let dt = unit - .datetime(obj.value, self.opts) - .map_err(NumpyDateTimeError::into_serde_err)?; - dt.serialize(serializer) - } else { - unreachable!() - } - } - } -} - -#[repr(C)] -pub struct NumpyInt8 { - ob_refcnt: Py_ssize_t, - ob_type: *mut PyTypeObject, - value: i8, -} - -impl Serialize for NumpyInt8 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_i8(self.value) - } -} - -#[repr(C)] -pub struct NumpyInt32 { - ob_refcnt: Py_ssize_t, - ob_type: *mut PyTypeObject, - value: i32, -} - -impl Serialize for NumpyInt32 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_i32(self.value) - } -} - -#[repr(C)] -pub struct NumpyInt64 { - ob_refcnt: Py_ssize_t, - ob_type: *mut PyTypeObject, - value: i64, -} - -impl Serialize for NumpyInt64 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_i64(self.value) - } -} - -#[repr(C)] -pub struct NumpyUint8 { - ob_refcnt: Py_ssize_t, - ob_type: *mut PyTypeObject, - value: u8, -} - -impl Serialize for NumpyUint8 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_u8(self.value) - } -} - -#[repr(C)] -pub struct NumpyUint32 { - ob_refcnt: Py_ssize_t, - ob_type: *mut PyTypeObject, - value: u32, -} - -impl Serialize for NumpyUint32 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_u32(self.value) - } -} - -#[repr(C)] -pub struct NumpyUint64 { - ob_refcnt: Py_ssize_t, - ob_type: *mut PyTypeObject, - value: u64, -} - -impl Serialize for NumpyUint64 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_u64(self.value) - } -} - -#[repr(C)] -pub struct NumpyFloat32 { - ob_refcnt: Py_ssize_t, - ob_type: *mut PyTypeObject, - value: f32, -} - -impl Serialize for NumpyFloat32 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_f32(self.value) - } -} - -#[repr(C)] -pub struct NumpyFloat64 { - ob_refcnt: Py_ssize_t, - ob_type: *mut PyTypeObject, - value: f64, -} - -impl Serialize for NumpyFloat64 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_f64(self.value) - } -} - -#[repr(C)] -pub struct NumpyBool { - ob_refcnt: Py_ssize_t, - ob_type: *mut PyTypeObject, - value: bool, -} - -impl Serialize for NumpyBool { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_bool(self.value) - } -} - -/// This mimicks the units supported by numpy's datetime64 type. -/// -/// See -/// https://github.com/numpy/numpy/blob/fc8e3bbe419748ac5c6b7f3d0845e4bafa74644b/numpy/core/include/numpy/ndarraytypes.h#L268-L282. -#[derive(Clone, Copy, Debug)] -pub enum NumpyDatetimeUnit { - NaT, - Years, - Months, - Weeks, - Days, - Hours, - Minutes, - Seconds, - Milliseconds, - Microseconds, - Nanoseconds, - Picoseconds, - Femtoseconds, - Attoseconds, - Generic, -} - -impl fmt::Display for NumpyDatetimeUnit { - #[cold] - #[cfg_attr(feature = "unstable-simd", optimize(size))] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let unit = match self { - Self::NaT => "NaT", - Self::Years => "years", - Self::Months => "months", - Self::Weeks => "weeks", - Self::Days => "days", - Self::Hours => "hours", - Self::Minutes => "minutes", - Self::Seconds => "seconds", - Self::Milliseconds => "milliseconds", - Self::Microseconds => "microseconds", - Self::Nanoseconds => "nanoseconds", - Self::Picoseconds => "picoseconds", - Self::Femtoseconds => "femtoseconds", - Self::Attoseconds => "attoseconds", - Self::Generic => "generic", - }; - write!(f, "{}", unit) - } -} - -#[derive(Clone, Copy, Debug)] -enum NumpyDateTimeError { - UnsupportedUnit(NumpyDatetimeUnit), - Unrepresentable { unit: NumpyDatetimeUnit, val: i64 }, -} - -impl NumpyDateTimeError { - #[cold] - #[cfg_attr(feature = "unstable-simd", optimize(size))] - fn into_serde_err(self) -> T { - let err = match self { - Self::UnsupportedUnit(unit) => format!("unsupported numpy.datetime64 unit: {}", unit), - Self::Unrepresentable { unit, val } => { - format!("unrepresentable numpy.datetime64: {} {}", val, unit) - } - }; - ser::Error::custom(err) - } -} - -impl NumpyDatetimeUnit { - /// Create a `NumpyDatetimeUnit` from a pointer to a Python object holding a - /// numpy array. - /// - /// This function must only be called with pointers to numpy arrays. - /// - /// We need to look inside the `obj.dtype.descr` attribute of the Python - /// object rather than using the `descr` field of the `__array_struct__` - /// because that field isn't populated for datetime64 arrays; see - /// https://github.com/numpy/numpy/issues/5350. - fn from_pyobject(ptr: *mut PyObject) -> Self { - let dtype = ffi!(PyObject_GetAttr(ptr, DTYPE_STR)); - let descr = ffi!(PyObject_GetAttr(dtype, DESCR_STR)); - ffi!(Py_DECREF(dtype)); - let el0 = ffi!(PyList_GET_ITEM(descr, 0)); - ffi!(Py_DECREF(descr)); - let descr_str = ffi!(PyTuple_GET_ITEM(el0, 1)); - let uni = crate::unicode::unicode_to_str(descr_str).unwrap(); - if uni.len() < 5 { - return Self::NaT; - } - // unit descriptions are found at - // https://github.com/numpy/numpy/blob/b235f9e701e14ed6f6f6dcba885f7986a833743f/numpy/core/src/multiarray/datetime.c#L79-L96. - match &uni[4..uni.len() - 1] { - "Y" => Self::Years, - "M" => Self::Months, - "W" => Self::Weeks, - "D" => Self::Days, - "h" => Self::Hours, - "m" => Self::Minutes, - "s" => Self::Seconds, - "ms" => Self::Milliseconds, - "us" => Self::Microseconds, - "ns" => Self::Nanoseconds, - "ps" => Self::Picoseconds, - "fs" => Self::Femtoseconds, - "as" => Self::Attoseconds, - "generic" => Self::Generic, - _ => unreachable!(), - } - } - - /// Return a `NumpyDatetime64Repr` for a value in array with this unit. - /// - /// Returns an `Err(NumpyDateTimeError)` if the value is invalid for this unit. - fn datetime(&self, val: i64, opts: Opt) -> Result { - match self { - Self::Years => Ok(NaiveDate::from_ymd( - (val + 1970) - .try_into() - .map_err(|_| NumpyDateTimeError::Unrepresentable { unit: *self, val })?, - 1, - 1, - ) - .and_hms(0, 0, 0)), - Self::Months => Ok(NaiveDate::from_ymd( - (val / 12 + 1970) - .try_into() - .map_err(|_| NumpyDateTimeError::Unrepresentable { unit: *self, val })?, - (val % 12 + 1) - .try_into() - .map_err(|_| NumpyDateTimeError::Unrepresentable { unit: *self, val })?, - 1, - ) - .and_hms(0, 0, 0)), - Self::Weeks => Ok(NaiveDateTime::from_timestamp(val * 7 * 24 * 60 * 60, 0)), - Self::Days => Ok(NaiveDateTime::from_timestamp(val * 24 * 60 * 60, 0)), - Self::Hours => Ok(NaiveDateTime::from_timestamp(val * 60 * 60, 0)), - Self::Minutes => Ok(NaiveDateTime::from_timestamp(val * 60, 0)), - Self::Seconds => Ok(NaiveDateTime::from_timestamp(val, 0)), - Self::Milliseconds => Ok(NaiveDateTime::from_timestamp( - val / 1_000, - (val % 1_000 * 1_000_000) - .try_into() - .map_err(|_| NumpyDateTimeError::Unrepresentable { unit: *self, val })?, - )), - Self::Microseconds => Ok(NaiveDateTime::from_timestamp( - val / 1_000_000, - (val % 1_000_000 * 1_000) - .try_into() - .map_err(|_| NumpyDateTimeError::Unrepresentable { unit: *self, val })?, - )), - Self::Nanoseconds => Ok(NaiveDateTime::from_timestamp( - val / 1_000_000_000, - (val % 1_000_000_000) - .try_into() - .map_err(|_| NumpyDateTimeError::Unrepresentable { unit: *self, val })?, - )), - _ => Err(NumpyDateTimeError::UnsupportedUnit(*self)), - } - .map(|dt| NumpyDatetime64Repr { dt, opts }) - } -} - -#[repr(C)] -pub struct NumpyDatetime64 { - ob_refcnt: Py_ssize_t, - ob_type: *mut PyTypeObject, - value: i64, -} - -macro_rules! forward_inner { - ($meth: ident, $ty: ident) => { - fn $meth(&self) -> $ty { - self.dt.$meth() as $ty - } - }; -} - -struct NumpyDatetime64Repr { - dt: NaiveDateTime, - opts: Opt, -} - -impl DateTimeLike for NumpyDatetime64Repr { - forward_inner!(year, i32); - forward_inner!(month, u8); - forward_inner!(day, u8); - forward_inner!(hour, u8); - forward_inner!(minute, u8); - forward_inner!(second, u8); - forward_inner!(nanosecond, u32); - - fn millisecond(&self) -> u32 { - self.nanosecond() / 1_000_000 - } - - fn microsecond(&self) -> u32 { - self.nanosecond() / 1_000 - } - - fn has_tz(&self) -> bool { - false - } - - fn slow_offset(&self) -> Result { - unreachable!() - } - - fn offset(&self) -> Result { - Ok(Offset::default()) - } -} - -impl Serialize for NumpyDatetime64Repr { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut buf = DateTimeBuffer::new(); - self.write_buf(&mut buf, self.opts).unwrap(); - serializer.collect_str(str_from_slice!(buf.as_ptr(), buf.len())) - } -} diff --git a/src/serialize/numpy/array.rs b/src/serialize/numpy/array.rs new file mode 100644 index 00000000..47684d79 --- /dev/null +++ b/src/serialize/numpy/array.rs @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright ijl (2020-2026), Ben Sully (2021) + +use super::item::ItemType; +use crate::ffi::{ + NPY_ARRAY_C_CONTIGUOUS, NPY_ARRAY_NOTSWAPPED, NumpyDatetimeUnit, Py_DECREF, PyArrayInterface, + PyCapsule, PyObject, PyObject_GetAttr, +}; +use crate::opt::Opt; +use crate::typeref::ARRAY_STRUCT_STR; +use crate::util::isize_to_usize; +use core::ffi::c_void; + +macro_rules! slice { + ($ptr:expr, $size:expr) => { + unsafe { core::slice::from_raw_parts($ptr, $size) } + }; +} + +pub(crate) enum PyArrayError { + Malformed, + NotContiguous, + NotNativeEndian, + UnsupportedDataType, +} + +// >>> arr = numpy.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]], numpy.int32) +// >>> arr.ndim +// 3 +// >>> arr.shape +// (2, 2, 2) +// >>> arr.strides +// (16, 8, 4) +pub(crate) struct NumpyArray { + array: *mut PyArrayInterface, + unit: Option, + position: Vec, + pub children: Vec, + pub depth: usize, + capsule: *mut PyCapsule, + pub kind: ItemType, + pub opts: Opt, +} + +impl NumpyArray { + #[cold] + #[inline(never)] + #[cfg_attr(feature = "optimize", optimize(size))] + pub fn new(ptr: *mut PyObject, opts: Opt) -> Result { + let capsule = unsafe { PyObject_GetAttr(ptr, ARRAY_STRUCT_STR) }; + debug_assert!(!capsule.is_null()); + let array = unsafe { + (*capsule.cast::()) + .pointer + .cast::() + }; + debug_assert!(!array.is_null()); + if unsafe { (*array).two != 2 } { + unsafe { + Py_DECREF(capsule); + } + Err(PyArrayError::Malformed) + } else if unsafe { (*array).flags } & NPY_ARRAY_C_CONTIGUOUS != NPY_ARRAY_C_CONTIGUOUS { + unsafe { + Py_DECREF(capsule); + } + Err(PyArrayError::NotContiguous) + } else if unsafe { (*array).flags } & NPY_ARRAY_NOTSWAPPED != NPY_ARRAY_NOTSWAPPED { + unsafe { + Py_DECREF(capsule); + } + Err(PyArrayError::NotNativeEndian) + } else { + debug_assert!(unsafe { (*array).nd >= 0 }); + #[allow(clippy::cast_sign_loss)] + let num_dimensions = unsafe { (*array).nd as usize }; + if num_dimensions == 0 { + unsafe { + Py_DECREF(capsule); + } + return Err(PyArrayError::UnsupportedDataType); + } + match ItemType::find(array, ptr) { + None => { + unsafe { + Py_DECREF(capsule); + } + Err(PyArrayError::UnsupportedDataType) + } + Some(kind) => { + let unit = match kind { + ItemType::DATETIME64(val) => Some(val), + _ => None, + }; + let mut pyarray = NumpyArray { + array: array, + unit: unit, + position: vec![0; num_dimensions], + children: Vec::with_capacity(num_dimensions), + depth: 0, + capsule: capsule.cast::(), + kind: kind, + opts: opts, + }; + if pyarray.dimensions() > 1 { + pyarray.build(); + } + Ok(pyarray) + } + } + } + } + + fn child_from_parent(&self, position: Vec, num_children: usize) -> Self { + let mut arr = NumpyArray { + array: self.array, + unit: self.unit, + position: position, + children: Vec::with_capacity(num_children), + depth: self.depth + 1, + capsule: self.capsule, + kind: self.kind, + opts: self.opts, + }; + arr.build(); + arr + } + + pub fn build(&mut self) { + if self.depth < self.dimensions() - 1 { + for i in 0..self.shape()[self.depth] { + let mut position: Vec = self.position.clone(); + position[self.depth] = i; + let num_children: usize = if self.depth < self.dimensions() - 2 { + isize_to_usize(self.shape()[self.depth + 1]) + } else { + 0 + }; + self.children + .push(self.child_from_parent(position, num_children)); + } + } + } + + #[inline(always)] + pub fn data(&self) -> *const c_void { + let offset = self + .strides() + .iter() + .zip(self.position.iter().copied()) + .take(self.depth) + .map(|(a, b)| a * b) + .sum::(); + unsafe { (*self.array).data.offset(offset) } + } + + pub fn num_items(&self) -> usize { + isize_to_usize(self.shape()[self.shape().len() - 1]) + } + + pub fn dimensions(&self) -> usize { + #[allow(clippy::cast_sign_loss)] + unsafe { + (*self.array).nd as usize + } + } + + pub fn shape(&self) -> &[isize] { + slice!((*self.array).shape.cast_const(), self.dimensions()) + } + + pub fn strides(&self) -> &[isize] { + slice!((*self.array).strides.cast_const(), self.dimensions()) + } +} + +impl Drop for NumpyArray { + fn drop(&mut self) { + if self.depth == 0 { + unsafe { + Py_DECREF(self.array.cast::()); + Py_DECREF(self.capsule.cast::()); + }; + } + } +} + +#[repr(transparent)] +pub(crate) struct NumpyF64Array<'a> { + pub data: &'a [f64], +} + +impl<'a> NumpyF64Array<'a> { + pub const fn new(data: &'a [f64]) -> Self { + Self { data } + } +} + +#[repr(transparent)] +pub(crate) struct NumpyF32Array<'a> { + pub data: &'a [f32], +} + +impl<'a> NumpyF32Array<'a> { + pub const fn new(data: &'a [f32]) -> Self { + Self { data } + } +} + +#[repr(transparent)] +pub(crate) struct NumpyF16Array<'a> { + pub data: &'a [u16], +} + +impl<'a> NumpyF16Array<'a> { + pub const fn new(data: &'a [u16]) -> Self { + Self { data } + } +} + +#[repr(transparent)] +pub(crate) struct NumpyU64Array<'a> { + pub data: &'a [u64], +} + +impl<'a> NumpyU64Array<'a> { + pub const fn new(data: &'a [u64]) -> Self { + Self { data } + } +} + +#[repr(transparent)] +pub(crate) struct NumpyU32Array<'a> { + pub data: &'a [u32], +} + +impl<'a> NumpyU32Array<'a> { + pub const fn new(data: &'a [u32]) -> Self { + Self { data } + } +} + +#[repr(transparent)] +pub(crate) struct NumpyU16Array<'a> { + pub data: &'a [u16], +} + +impl<'a> NumpyU16Array<'a> { + pub const fn new(data: &'a [u16]) -> Self { + Self { data } + } +} + +#[repr(transparent)] +pub(crate) struct NumpyU8Array<'a> { + pub data: &'a [u8], +} + +impl<'a> NumpyU8Array<'a> { + pub const fn new(data: &'a [u8]) -> Self { + Self { data } + } +} + +#[repr(transparent)] +pub(crate) struct NumpyI64Array<'a> { + pub data: &'a [i64], +} + +impl<'a> NumpyI64Array<'a> { + pub const fn new(data: &'a [i64]) -> Self { + Self { data } + } +} + +#[repr(transparent)] +pub(crate) struct NumpyI32Array<'a> { + pub data: &'a [i32], +} + +impl<'a> NumpyI32Array<'a> { + pub const fn new(data: &'a [i32]) -> Self { + Self { data } + } +} + +#[repr(transparent)] +pub(crate) struct NumpyI16Array<'a> { + pub data: &'a [i16], +} + +impl<'a> NumpyI16Array<'a> { + pub const fn new(data: &'a [i16]) -> Self { + Self { data } + } +} + +#[repr(transparent)] +pub(crate) struct NumpyI8Array<'a> { + pub data: &'a [i8], +} + +impl<'a> NumpyI8Array<'a> { + pub const fn new(data: &'a [i8]) -> Self { + Self { data } + } +} + +#[repr(transparent)] +pub(crate) struct NumpyBoolArray<'a> { + pub data: &'a [u8], +} + +impl<'a> NumpyBoolArray<'a> { + pub const fn new(data: &'a [u8]) -> Self { + Self { data } + } +} + +pub(crate) struct NumpyDatetime64Array<'a> { + pub data: &'a [i64], + pub unit: NumpyDatetimeUnit, + pub opts: Opt, +} + +impl<'a> NumpyDatetime64Array<'a> { + pub const fn new(data: &'a [i64], unit: NumpyDatetimeUnit, opts: Opt) -> Self { + Self { data, unit, opts } + } +} diff --git a/src/serialize/numpy/datetime.rs b/src/serialize/numpy/datetime.rs new file mode 100644 index 00000000..2bbdde21 --- /dev/null +++ b/src/serialize/numpy/datetime.rs @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +use crate::ffi::{NumpyDateTimeError, NumpyDatetime64Repr, PyStrRef}; +use crate::opt::{NAIVE_UTC, OMIT_MICROSECONDS, UTC_Z}; +use crate::serialize::{ + error::SerializeError, + writer::{SmallFixedBuffer, WriteExt, write_integer_i64, write_integer_u32}, +}; + +pub(crate) fn datetime_into_error(val: NumpyDateTimeError) -> SerializeError { + let err = match val { + NumpyDateTimeError::UnsupportedUnit(unit) => { + let mut msg = String::from("unsupported numpy.datetime64 unit: "); + msg.push_str(unit.as_str()); + msg + } + NumpyDateTimeError::Unrepresentable { unit, val } => { + let mut buf = SmallFixedBuffer::new(); + write_integer_i64(&mut buf, val); + let val_str = str_from_slice!(buf.as_ptr(), buf.len()); + + let mut msg = String::from("unrepresentable numpy.datetime64: "); + msg.push_str(val_str); + msg.push(' '); + msg.push_str(unit.as_str()); + msg + } + }; + SerializeError::NumpyUnsupportedDatetimeUnit(PyStrRef::from_str(&err)) +} + +pub(crate) fn write_numpy_datetime(ob: &NumpyDatetime64Repr, buf: &mut B) +where + B: ?Sized + WriteExt + bytes::BufMut, +{ + { + // buf.put_u8(b'"'); + let year = ob.year(); + if year < 1000 { + cold_path!(); + // date-fullyear = 4DIGIT + buf.put_u8(b'0'); + if year < 100 { + buf.put_u8(b'0'); + } + if year < 10 { + buf.put_u8(b'0'); + } + } + write_integer_u32(buf, year.cast_unsigned()); + } + buf.put_u8(b'-'); + write_double_digit!(buf, ob.month() as u32); + buf.put_u8(b'-'); + write_double_digit!(buf, ob.day() as u32); + buf.put_u8(b'T'); + write_double_digit!(buf, ob.hour() as u32); + buf.put_u8(b':'); + write_double_digit!(buf, ob.minute() as u32); + buf.put_u8(b':'); + write_double_digit!(buf, ob.second() as u32); + if opt_disabled!(ob.opts, OMIT_MICROSECONDS) { + let microsecond = ob.microsecond(); + if microsecond != 0 { + buf.put_u8(b'.'); + write_triple_digit!(buf, microsecond / 1_000); + write_triple_digit!(buf, microsecond % 1_000); + } + } + if opt_enabled!(ob.opts, NAIVE_UTC) { + if opt_enabled!(ob.opts, UTC_Z) { + buf.put_u8(b'Z'); + } else { + buf.put_slice(b"+00:00"); + } + } + // buf.put_u8(b'"'); +} diff --git a/src/serialize/numpy/item.rs b/src/serialize/numpy/item.rs new file mode 100644 index 00000000..10ac99a1 --- /dev/null +++ b/src/serialize/numpy/item.rs @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright ijl (2020-2026), Nazar Kostetskyi (2022), Ben Sully (2021) + +use crate::ffi::{NumpyDatetimeUnit, PyArrayInterface, PyObject}; + +#[derive(Clone, Copy)] +pub(crate) enum ItemType { + BOOL, + DATETIME64(NumpyDatetimeUnit), + F16, + F32, + F64, + I8, + I16, + I32, + I64, + U8, + U16, + U32, + U64, +} + +impl ItemType { + pub fn find(array: *mut PyArrayInterface, ptr: *mut PyObject) -> Option { + match unsafe { ((*array).typekind, (*array).itemsize) } { + (098, 1) => Some(ItemType::BOOL), + (077, 8) => { + let unit = NumpyDatetimeUnit::from_pyobject(ptr); + Some(ItemType::DATETIME64(unit)) + } + (102, 2) => Some(ItemType::F16), + (102, 4) => Some(ItemType::F32), + (102, 8) => Some(ItemType::F64), + (105, 1) => Some(ItemType::I8), + (105, 2) => Some(ItemType::I16), + (105, 4) => Some(ItemType::I32), + (105, 8) => Some(ItemType::I64), + (117, 1) => Some(ItemType::U8), + (117, 2) => Some(ItemType::U16), + (117, 4) => Some(ItemType::U32), + (117, 8) => Some(ItemType::U64), + _ => None, + } + } +} diff --git a/src/serialize/numpy/mod.rs b/src/serialize/numpy/mod.rs new file mode 100644 index 00000000..78c82cc0 --- /dev/null +++ b/src/serialize/numpy/mod.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +mod array; +mod datetime; +mod item; +mod scalar; + +pub(crate) use array::*; +pub(crate) use datetime::*; +pub(crate) use item::*; +pub(crate) use scalar::*; diff --git a/src/serialize/numpy/scalar.rs b/src/serialize/numpy/scalar.rs new file mode 100644 index 00000000..2bb6e89f --- /dev/null +++ b/src/serialize/numpy/scalar.rs @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +use crate::ffi::PyObject; +use crate::opt::Opt; + +pub(crate) struct NumpyScalar { + pub ptr: *mut PyObject, + pub opts: Opt, +} + +impl NumpyScalar { + pub const fn new(ptr: *mut PyObject, opts: Opt) -> Self { + NumpyScalar { ptr, opts } + } +} diff --git a/src/serialize/obtype.rs b/src/serialize/obtype.rs new file mode 100644 index 00000000..e792c4ea --- /dev/null +++ b/src/serialize/obtype.rs @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright ijl (2020-2026), Aviram Hassan (2020) + +use crate::ffi::PyType_GetFlags; +use crate::opt::{ + Opt, PASSTHROUGH_DATACLASS, PASSTHROUGH_DATETIME, PASSTHROUGH_SUBCLASS, SERIALIZE_NUMPY, +}; +use crate::serialize::per_type::{is_numpy_array, is_numpy_scalar}; +use crate::typeref::{ + BOOL_TYPE, DATACLASS_FIELDS_STR, DATE_TYPE, DATETIME_TYPE, DICT_TYPE, ENUM_TYPE, FLOAT_TYPE, + FRAGMENT_TYPE, INT_TYPE, LIST_TYPE, NONE_TYPE, STR_TYPE, TIME_TYPE, TUPLE_TYPE, UUID_TYPE, +}; + +#[repr(u32)] +pub(crate) enum ObType { + Str, + Int, + Bool, + None, + Float, + List, + Dict, + Datetime, + Date, + Time, + Tuple, + Uuid, + Dataclass, + NumpyScalar, + NumpyArray, + Enum, + StrSubclass, + Fragment, + Unknown, +} + +pub(crate) fn pyobject_to_obtype(obj: *mut crate::ffi::PyObject, opts: Opt) -> ObType { + let ob_type = unsafe { crate::ffi::PyObject_Type(obj) }; + if is_class_by_type!(ob_type, STR_TYPE) { + ObType::Str + } else if is_class_by_type!(ob_type, INT_TYPE) { + ObType::Int + } else if is_class_by_type!(ob_type, BOOL_TYPE) { + ObType::Bool + } else if is_class_by_type!(ob_type, NONE_TYPE) { + ObType::None + } else if is_class_by_type!(ob_type, FLOAT_TYPE) { + ObType::Float + } else if is_class_by_type!(ob_type, LIST_TYPE) { + ObType::List + } else if is_class_by_type!(ob_type, DICT_TYPE) { + ObType::Dict + } else if is_class_by_type!(ob_type, DATETIME_TYPE) && opt_disabled!(opts, PASSTHROUGH_DATETIME) + { + ObType::Datetime + } else { + pyobject_to_obtype_unlikely(ob_type, opts) + } +} + +#[cfg_attr(feature = "optimize", optimize(size))] +#[inline(never)] +pub(crate) fn pyobject_to_obtype_unlikely( + ob_type: *mut crate::ffi::PyTypeObject, + opts: Opt, +) -> ObType { + if is_class_by_type!(ob_type, UUID_TYPE) { + return ObType::Uuid; + } else if is_class_by_type!(ob_type, TUPLE_TYPE) { + return ObType::Tuple; + } else if is_class_by_type!(ob_type, FRAGMENT_TYPE) { + return ObType::Fragment; + } + + if opt_disabled!(opts, PASSTHROUGH_DATETIME) { + if is_class_by_type!(ob_type, DATE_TYPE) { + return ObType::Date; + } else if is_class_by_type!(ob_type, TIME_TYPE) { + return ObType::Time; + } + } + + let tp_flags = unsafe { PyType_GetFlags(ob_type) }; + + if opt_disabled!(opts, PASSTHROUGH_SUBCLASS) { + if is_subclass_by_flag!(tp_flags, Py_TPFLAGS_UNICODE_SUBCLASS) { + return ObType::StrSubclass; + } else if is_subclass_by_flag!(tp_flags, Py_TPFLAGS_LONG_SUBCLASS) { + return ObType::Int; + } else if is_subclass_by_flag!(tp_flags, Py_TPFLAGS_LIST_SUBCLASS) { + return ObType::List; + } else if is_subclass_by_flag!(tp_flags, Py_TPFLAGS_DICT_SUBCLASS) { + return ObType::Dict; + } + } + + if is_subclass_by_type!(ob_type, ENUM_TYPE) { + return ObType::Enum; + } + + if opt_disabled!(opts, PASSTHROUGH_DATACLASS) && pydict_contains!(ob_type, DATACLASS_FIELDS_STR) + { + return ObType::Dataclass; + } + + if opt_enabled!(opts, SERIALIZE_NUMPY) { + cold_path!(); + if is_numpy_scalar(ob_type) { + return ObType::NumpyScalar; + } else if is_numpy_array(ob_type) { + return ObType::NumpyArray; + } + } + + ObType::Unknown +} diff --git a/src/serialize/per_type/dataclass.rs b/src/serialize/per_type/dataclass.rs new file mode 100644 index 00000000..2aea1eb4 --- /dev/null +++ b/src/serialize/per_type/dataclass.rs @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2018-2026) + +use crate::ffi::PyStrRef; +use crate::serialize::error::SerializeError; +use crate::serialize::per_type::dict::ZeroDictSerializer; +use crate::serialize::serializer::PyObjectSerializer; +use crate::serialize::state::SerializerState; +use crate::typeref::{ + DATACLASS_FIELDS_STR, DICT_STR, FIELD_TYPE, FIELD_TYPE_STR, SLOTS_STR, STR_TYPE, +}; +use crate::util::isize_to_usize; + +use serde::ser::{Serialize, SerializeMap, Serializer}; + +use core::ptr::NonNull; + +#[repr(transparent)] +pub(crate) struct DataclassGenericSerializer<'a> { + previous: &'a PyObjectSerializer, +} + +impl<'a> DataclassGenericSerializer<'a> { + pub fn new(previous: &'a PyObjectSerializer) -> Self { + Self { previous: previous } + } +} + +impl Serialize for DataclassGenericSerializer<'_> { + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + if self.previous.state.recursion_limit() { + err!(SerializeError::RecursionLimit) + } + let dict = ffi!(PyObject_GetAttr(self.previous.ptr, DICT_STR)); + let ob_type = unsafe { crate::ffi::PyObject_Type(self.previous.ptr) }; + if dict.is_null() { + cold_path!(); + ffi!(PyErr_Clear()); + DataclassFallbackSerializer::new( + self.previous.ptr, + self.previous.state, + self.previous.default, + ) + .serialize(serializer) + } else if pydict_contains!(ob_type, SLOTS_STR) { + let ret = DataclassFallbackSerializer::new( + self.previous.ptr, + self.previous.state, + self.previous.default, + ) + .serialize(serializer); + ffi!(Py_DECREF(dict)); + ret + } else { + let ret = + DataclassFastSerializer::new(dict, self.previous.state, self.previous.default) + .serialize(serializer); + ffi!(Py_DECREF(dict)); + ret + } + } +} + +pub(crate) struct DataclassFastSerializer { + ptr: *mut crate::ffi::PyObject, + state: SerializerState, + default: Option>, +} + +impl DataclassFastSerializer { + pub fn new( + ptr: *mut crate::ffi::PyObject, + state: SerializerState, + default: Option>, + ) -> Self { + DataclassFastSerializer { + ptr: ptr, + state: state.copy_for_recursive_call(), + default: default, + } + } +} + +impl Serialize for DataclassFastSerializer { + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let len = isize_to_usize(ffi!(Py_SIZE(self.ptr))); + if len == 0 { + cold_path!(); + return ZeroDictSerializer::new().serialize(serializer); + } + let mut map = serializer.serialize_map(None).unwrap(); + + let mut pos = 0; + let mut next_key: *mut crate::ffi::PyObject = core::ptr::null_mut(); + let mut next_value: *mut crate::ffi::PyObject = core::ptr::null_mut(); + + unsafe { + crate::ffi::PyDict_Next( + self.ptr, + &raw mut pos, + &raw mut next_key, + &raw mut next_value, + ); + } + + for _ in 0..len { + let key = next_key; + let value = next_value; + + unsafe { + crate::ffi::PyDict_Next( + self.ptr, + &raw mut pos, + &raw mut next_key, + &raw mut next_value, + ); + } + + let key_as_str = { + let key_ob_type = unsafe { crate::ffi::PyObject_Type(key) }; + if !is_class_by_type!(key_ob_type, STR_TYPE) { + cold_path!(); + err!(SerializeError::KeyMustBeStr) + } + match unsafe { PyStrRef::from_ptr_unchecked(key).as_str() } { + Some(uni) => uni, + None => err!(SerializeError::InvalidStr), + } + }; + if key_as_str.as_bytes()[0] == b'_' { + cold_path!(); + continue; + } + let pyvalue = PyObjectSerializer::new(value, self.state, self.default); + map.serialize_key(key_as_str).unwrap(); + map.serialize_value(&pyvalue)?; + } + map.end() + } +} + +pub(crate) struct DataclassFallbackSerializer { + ptr: *mut crate::ffi::PyObject, + state: SerializerState, + default: Option>, +} + +impl DataclassFallbackSerializer { + pub fn new( + ptr: *mut crate::ffi::PyObject, + state: SerializerState, + default: Option>, + ) -> Self { + DataclassFallbackSerializer { + ptr: ptr, + state: state.copy_for_recursive_call(), + default: default, + } + } +} + +impl Serialize for DataclassFallbackSerializer { + #[cold] + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let fields = ffi!(PyObject_GetAttr(self.ptr, DATACLASS_FIELDS_STR)); + debug_assert!(ffi!(Py_REFCNT(fields)) >= 2); + ffi!(Py_DECREF(fields)); + let len = isize_to_usize(ffi!(Py_SIZE(fields))); + if len == 0 { + cold_path!(); + return ZeroDictSerializer::new().serialize(serializer); + } + let mut map = serializer.serialize_map(None).unwrap(); + + let mut pos = 0; + let mut next_key: *mut crate::ffi::PyObject = core::ptr::null_mut(); + let mut next_value: *mut crate::ffi::PyObject = core::ptr::null_mut(); + + unsafe { + crate::ffi::PyDict_Next(fields, &raw mut pos, &raw mut next_key, &raw mut next_value); + } + + for _ in 0..len { + let attr = next_key; + let field = next_value; + unsafe { + crate::ffi::PyDict_Next( + fields, + &raw mut pos, + &raw mut next_key, + &raw mut next_value, + ); + } + + let field_type = ffi!(PyObject_GetAttr(field, FIELD_TYPE_STR)); + debug_assert!(ffi!(Py_REFCNT(field_type)) >= 2); + ffi!(Py_DECREF(field_type)); + if unsafe { !core::ptr::eq(field_type.cast::(), FIELD_TYPE) } + { + continue; + } + + let key_as_str = match unsafe { PyStrRef::from_ptr_unchecked(attr).as_str() } { + Some(uni) => uni, + None => err!(SerializeError::InvalidStr), + }; + if key_as_str.as_bytes()[0] == b'_' { + cold_path!(); + continue; + } + + let value = ffi!(PyObject_GetAttr(self.ptr, attr)); + debug_assert!(ffi!(Py_REFCNT(value)) >= 2); + ffi!(Py_DECREF(value)); + let pyvalue = PyObjectSerializer::new(value, self.state, self.default); + + map.serialize_key(key_as_str).unwrap(); + map.serialize_value(&pyvalue)?; + } + map.end() + } +} diff --git a/src/serialize/per_type/datetime.rs b/src/serialize/per_type/datetime.rs new file mode 100644 index 00000000..8a3b9e6d --- /dev/null +++ b/src/serialize/per_type/datetime.rs @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2018-2026) + +use crate::ffi::{PyDateRef, PyDateTimeRef, PyTimeRef}; +use crate::opt::Opt; +use crate::serialize::datetime::{write_date, write_datetime, write_time}; +use crate::serialize::error::SerializeError; +use crate::serialize::writer::SmallFixedBuffer; +use serde::ser::{Serialize, Serializer}; + +#[repr(transparent)] +pub(crate) struct Date { + ob: PyDateRef, +} + +impl Date { + pub fn new(ob: PyDateRef) -> Self { + Date { ob } + } +} +impl Serialize for Date { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut buf = SmallFixedBuffer::new(); + write_date(self.ob.clone(), &mut buf); + serializer.serialize_unit_struct(str_from_slice!(buf.as_ptr(), buf.len())) + } +} + +pub(crate) struct Time { + ob: PyTimeRef, + opts: Opt, +} + +impl Time { + pub fn new(ob: PyTimeRef, opts: Opt) -> Self { + Time { ob, opts } + } +} + +impl Serialize for Time { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut buf = SmallFixedBuffer::new(); + if write_time(self.ob.clone(), self.opts, &mut buf).is_err() { + err!(SerializeError::DatetimeLibraryUnsupported) + } + serializer.serialize_unit_struct(str_from_slice!(buf.as_ptr(), buf.len())) + } +} + +pub(crate) struct DateTime { + ob: PyDateTimeRef, + opts: Opt, +} + +impl DateTime { + pub fn new(ob: PyDateTimeRef, opts: Opt) -> Self { + DateTime { ob, opts } + } +} + +impl Serialize for DateTime { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut buf = SmallFixedBuffer::new(); + if write_datetime(self.ob.clone(), self.opts, &mut buf).is_err() { + err!(SerializeError::DatetimeLibraryUnsupported) + } + serializer.serialize_unit_struct(str_from_slice!(buf.as_ptr(), buf.len())) + } +} diff --git a/src/serialize/per_type/default.rs b/src/serialize/per_type/default.rs new file mode 100644 index 00000000..fb62eeb4 --- /dev/null +++ b/src/serialize/per_type/default.rs @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2018-2026) + +use crate::serialize::error::SerializeError; +use crate::serialize::serializer::PyObjectSerializer; + +use serde::ser::{Serialize, Serializer}; + +#[repr(transparent)] +pub(crate) struct DefaultSerializer<'a> { + previous: &'a PyObjectSerializer, +} + +impl<'a> DefaultSerializer<'a> { + pub fn new(previous: &'a PyObjectSerializer) -> Self { + Self { previous: previous } + } +} + +impl Serialize for DefaultSerializer<'_> { + #[cold] + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self.previous.default { + Some(callable) => { + if self.previous.state.default_calls_limit() { + cold_path!(); + err!(SerializeError::DefaultRecursionLimit) + } + let nargs = ffi!(PyVectorcall_NARGS(1)).cast_unsigned() as usize; + let default_obj = unsafe { + crate::ffi::PyObject_Vectorcall( + callable.as_ptr(), + &raw const self.previous.ptr, + nargs, + core::ptr::null_mut(), + ) + }; + if default_obj.is_null() { + err!(SerializeError::UnsupportedType(nonnull!(self.previous.ptr))) + } else { + let res = PyObjectSerializer::new( + default_obj, + self.previous.state.copy_for_default_call(), + self.previous.default, + ) + .serialize(serializer); + ffi!(Py_DECREF(default_obj)); + res + } + } + None => err!(SerializeError::UnsupportedType(nonnull!(self.previous.ptr))), + } + } +} diff --git a/src/serialize/per_type/dict.rs b/src/serialize/per_type/dict.rs new file mode 100644 index 00000000..71add104 --- /dev/null +++ b/src/serialize/per_type/dict.rs @@ -0,0 +1,586 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2018-2026) + +use crate::ffi::{ + PyBoolRef, PyDateRef, PyDateTimeRef, PyDictRef, PyFloatRef, PyFragmentRef, PyIntRef, PyListRef, + PyStrRef, PyStrSubclassRef, PyTimeRef, PyUuidRef, +}; +use crate::opt::{NON_STR_KEYS, NOT_PASSTHROUGH, SORT_KEYS, SORT_OR_NON_STR_KEYS}; +use crate::serialize::datetime::{write_date, write_datetime, write_time}; +use crate::serialize::error::SerializeError; +use crate::serialize::numpy::NumpyScalar; +use crate::serialize::obtype::{ObType, pyobject_to_obtype}; +use crate::serialize::per_type::{ + BoolSerializer, DataclassGenericSerializer, Date, DateTime, DefaultSerializer, EnumSerializer, + FloatSerializer, FragmentSerializer, IntSerializer, ListTupleSerializer, NoneSerializer, + NumpySerializer, StrSerializer, StrSubclassSerializer, Time, UUID, ZeroListSerializer, +}; +use crate::serialize::serializer::PyObjectSerializer; +use crate::serialize::state::SerializerState; +use crate::serialize::uuid::write_uuid; +use crate::serialize::writer::{SmallFixedBuffer, write_integer_i64, write_integer_u64}; +use crate::typeref::{STR_TYPE, TRUE, VALUE_STR}; +use core::ptr::NonNull; +use serde::ser::{Serialize, SerializeMap, Serializer}; + +pub(crate) struct ZeroDictSerializer; + +impl ZeroDictSerializer { + pub const fn new() -> Self { + Self {} + } +} + +impl Serialize for ZeroDictSerializer { + #[inline(always)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_bytes(b"{}") + } +} + +pub(crate) struct DictGenericSerializer { + dict: PyDictRef, + state: SerializerState, + #[allow(dead_code)] + default: Option>, +} + +impl DictGenericSerializer { + pub fn new( + dict: PyDictRef, + state: SerializerState, + default: Option>, + ) -> Self { + DictGenericSerializer { + dict: dict, + state: state.copy_for_recursive_call(), + default: default, + } + } +} + +impl Serialize for DictGenericSerializer { + #[inline(always)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + if self.state.recursion_limit() { + cold_path!(); + err!(SerializeError::RecursionLimit) + } + + if self.dict.len() == 0 { + cold_path!(); + ZeroDictSerializer::new().serialize(serializer) + } else if opt_disabled!(self.state.opts(), SORT_OR_NON_STR_KEYS) { + unsafe { + (*(core::ptr::from_ref::(self)).cast::()) + .serialize(serializer) + } + } else if opt_enabled!(self.state.opts(), NON_STR_KEYS) { + unsafe { + (*(core::ptr::from_ref::(self)).cast::()) + .serialize(serializer) + } + } else { + unsafe { + (*(core::ptr::from_ref::(self)).cast::()) + .serialize(serializer) + } + } + } +} + +macro_rules! impl_serialize_entry { + ($map:expr, $self:expr, $key:expr, $value:expr) => { + match pyobject_to_obtype($value, $self.state.opts()) { + ObType::Str => { + $map.serialize_key($key).unwrap(); + $map.serialize_value(&StrSerializer::new(unsafe { + PyStrRef::from_ptr_unchecked($value) + }))?; + } + ObType::StrSubclass => { + $map.serialize_key($key).unwrap(); + $map.serialize_value(&StrSubclassSerializer::new(unsafe { + PyStrSubclassRef::from_ptr_unchecked($value) + }))?; + } + ObType::Int => { + $map.serialize_key($key).unwrap(); + $map.serialize_value(&IntSerializer::new( + unsafe { PyIntRef::from_ptr_unchecked($value) }, + $self.state.opts(), + ))?; + } + ObType::None => { + $map.serialize_key($key).unwrap(); + $map.serialize_value(&NoneSerializer::new()).unwrap(); + } + ObType::Float => { + $map.serialize_key($key).unwrap(); + $map.serialize_value(&FloatSerializer::new(unsafe { + PyFloatRef::from_ptr_unchecked($value) + }))?; + } + ObType::Bool => { + $map.serialize_key($key).unwrap(); + $map.serialize_value(&BoolSerializer::new(unsafe { + PyBoolRef::from_ptr_unchecked($value) + })) + .unwrap(); + } + ObType::Datetime => { + $map.serialize_key($key).unwrap(); + $map.serialize_value(&DateTime::new( + unsafe { PyDateTimeRef::from_ptr_unchecked($value) }, + $self.state.opts(), + ))?; + } + ObType::Date => { + $map.serialize_key($key).unwrap(); + $map.serialize_value(&Date::new(unsafe { PyDateRef::from_ptr_unchecked($value) }))?; + } + ObType::Time => { + $map.serialize_key($key).unwrap(); + $map.serialize_value(&Time::new( + unsafe { PyTimeRef::from_ptr_unchecked($value) }, + $self.state.opts(), + ))?; + } + ObType::Uuid => { + $map.serialize_key($key).unwrap(); + $map.serialize_value(&UUID::new(unsafe { PyUuidRef::from_ptr_unchecked($value) })) + .unwrap(); + } + ObType::Dict => { + let pyvalue = DictGenericSerializer::new( + unsafe { PyDictRef::from_ptr_unchecked($value) }, + $self.state, + $self.default, + ); + $map.serialize_key($key).unwrap(); + $map.serialize_value(&pyvalue)?; + } + ObType::List => { + if ffi!(Py_SIZE($value)) == 0 { + $map.serialize_key($key).unwrap(); + $map.serialize_value(&ZeroListSerializer::new()).unwrap(); + } else { + let pyvalue = ListTupleSerializer::from_list( + unsafe { PyListRef::from_ptr_unchecked($value) }, + $self.state, + $self.default, + ); + $map.serialize_key($key).unwrap(); + $map.serialize_value(&pyvalue)?; + } + } + ObType::Tuple => { + if ffi!(Py_SIZE($value)) == 0 { + $map.serialize_key($key).unwrap(); + $map.serialize_value(&ZeroListSerializer::new()).unwrap(); + } else { + let pyvalue = + ListTupleSerializer::from_tuple($value, $self.state, $self.default); + $map.serialize_key($key).unwrap(); + $map.serialize_value(&pyvalue)?; + } + } + ObType::Dataclass => { + $map.serialize_key($key).unwrap(); + $map.serialize_value(&DataclassGenericSerializer::new(&PyObjectSerializer::new( + $value, + $self.state, + $self.default, + )))?; + } + ObType::Enum => { + $map.serialize_key($key).unwrap(); + $map.serialize_value(&EnumSerializer::new(&PyObjectSerializer::new( + $value, + $self.state, + $self.default, + )))?; + } + ObType::NumpyArray => { + $map.serialize_key($key).unwrap(); + $map.serialize_value(&NumpySerializer::new(&PyObjectSerializer::new( + $value, + $self.state, + $self.default, + )))?; + } + ObType::NumpyScalar => { + $map.serialize_key($key).unwrap(); + $map.serialize_value(&NumpyScalar::new($value, $self.state.opts()))?; + } + ObType::Fragment => { + $map.serialize_key($key).unwrap(); + $map.serialize_value(&FragmentSerializer::new(unsafe { + PyFragmentRef::from_ptr_unchecked($value) + }))?; + } + ObType::Unknown => { + $map.serialize_key($key).unwrap(); + $map.serialize_value(&DefaultSerializer::new(&PyObjectSerializer::new( + $value, + $self.state, + $self.default, + )))?; + } + } + }; +} + +pub(crate) struct Dict { + dict: PyDictRef, + state: SerializerState, + default: Option>, +} + +impl Serialize for Dict { + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut pos = 0; + let mut next_key: *mut crate::ffi::PyObject = core::ptr::null_mut(); + let mut next_value: *mut crate::ffi::PyObject = core::ptr::null_mut(); + unsafe { + crate::ffi::PyDict_Next( + self.dict.as_ptr(), + &raw mut pos, + &raw mut next_key, + &raw mut next_value, + ); + } + + let mut map = serializer.serialize_map(None).unwrap(); + + let len = self.dict.len(); + assume!(len > 0); + + for _ in 0..len { + let key = next_key; + let value = next_value; + + unsafe { + crate::ffi::PyDict_Next( + self.dict.as_ptr(), + &raw mut pos, + &raw mut next_key, + &raw mut next_value, + ); + } + + // key + let uni = PyStrRef::from_ptr(key) + .map_err(|_| serde::ser::Error::custom(SerializeError::KeyMustBeStr))? + .as_str(); + if uni.is_none() { + cold_path!(); + err!(SerializeError::InvalidStr); + } + + // value + impl_serialize_entry!(map, self, uni.unwrap(), value); + } + + map.end() + } +} + +pub(crate) struct DictSortedKey { + dict: PyDictRef, + state: SerializerState, + default: Option>, +} + +impl Serialize for DictSortedKey { + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut pos = 0; + let mut next_key: *mut crate::ffi::PyObject = core::ptr::null_mut(); + let mut next_value: *mut crate::ffi::PyObject = core::ptr::null_mut(); + + unsafe { + crate::ffi::PyDict_Next( + self.dict.as_ptr(), + &raw mut pos, + &raw mut next_key, + &raw mut next_value, + ); + } + + let len = self.dict.len(); + assume!(len > 0); + + let mut items: Vec<(&str, *mut crate::ffi::PyObject)> = Vec::with_capacity(len); + + for _ in 0..len { + let key = next_key; + let value = next_value; + unsafe { + crate::ffi::PyDict_Next( + self.dict.as_ptr(), + &raw mut pos, + &raw mut next_key, + &raw mut next_value, + ); + } + + if unsafe { !core::ptr::eq(crate::ffi::PyObject_Type(key), STR_TYPE) } { + err!(SerializeError::KeyMustBeStr) + } + let pystr = unsafe { PyStrRef::from_ptr_unchecked(key) }; + let uni = pystr.as_str(); + if uni.is_none() { + err!(SerializeError::InvalidStr) + } + let key_as_str = uni.unwrap(); + + items.push((key_as_str, value)); + } + + sort_dict_items(&mut items); + + let mut map = serializer.serialize_map(None).unwrap(); + for (key, val) in items.iter() { + let pyvalue = PyObjectSerializer::new(*val, self.state, self.default); + map.serialize_key(key).unwrap(); + map.serialize_value(&pyvalue)?; + } + map.end() + } +} +#[cold] +#[inline(never)] +fn non_str_str(key: PyStrRef) -> Result { + // because of ObType::Enum + match key.as_str() { + Some(uni) => Ok(String::from(uni)), + None => { + cold_path!(); + Err(SerializeError::InvalidStr) + } + } +} + +#[cold] +#[inline(never)] +fn non_str_str_subclass(key: PyStrSubclassRef) -> Result { + match key.as_str() { + Some(uni) => Ok(String::from(uni)), + None => { + cold_path!(); + Err(SerializeError::InvalidStr) + } + } +} + +#[allow(clippy::unnecessary_wraps)] +#[cold] +#[inline(never)] +fn non_str_date(key: PyDateRef) -> Result { + let mut buf = SmallFixedBuffer::new(); + write_date(key, &mut buf); + Ok(buf.to_string()) +} + +#[cold] +#[inline(never)] +fn non_str_datetime(key: PyDateTimeRef, opts: crate::opt::Opt) -> Result { + let mut buf = SmallFixedBuffer::new(); + if write_datetime(key, opts, &mut buf).is_err() { + return Err(SerializeError::DatetimeLibraryUnsupported); + } + Ok(buf.to_string()) +} + +#[cold] +#[inline(never)] +fn non_str_time(key: PyTimeRef, opts: crate::opt::Opt) -> Result { + let mut buf = SmallFixedBuffer::new(); + write_time(key, opts, &mut buf)?; + Ok(buf.to_string()) +} + +#[allow(clippy::unnecessary_wraps)] +#[inline(never)] +fn non_str_uuid(key: PyUuidRef) -> Result { + let mut buf = SmallFixedBuffer::new(); + write_uuid(key, &mut buf); + let key_as_str = str_from_slice!(buf.as_ptr(), buf.len()); + Ok(String::from(key_as_str)) +} + +#[allow(clippy::unnecessary_wraps)] +#[cold] +#[inline(never)] +fn non_str_float(key: *mut crate::ffi::PyObject) -> Result { + let val = ffi!(PyFloat_AS_DOUBLE(key)); + if !val.is_finite() { + Ok(String::from("null")) + } else { + Ok(String::from(zmij::Buffer::new().format_finite(val))) + } +} + +#[allow(clippy::unnecessary_wraps)] +#[inline(never)] +fn non_str_int(key: *mut crate::ffi::PyObject) -> Result { + let ival = ffi!(PyLong_AsLongLong(key)); + if ival == -1 && !ffi!(PyErr_Occurred()).is_null() { + cold_path!(); + ffi!(PyErr_Clear()); + let uval = ffi!(PyLong_AsUnsignedLongLong(key)); + if uval == u64::MAX && !ffi!(PyErr_Occurred()).is_null() { + cold_path!(); + return Err(SerializeError::DictIntegerKey64Bit); + } + let mut buf = SmallFixedBuffer::new(); + write_integer_u64(&mut buf, uval); + Ok(buf.to_string()) + } else { + let mut buf = SmallFixedBuffer::new(); + write_integer_i64(&mut buf, ival); + Ok(buf.to_string()) + } +} + +#[inline(never)] +fn sort_dict_items(items: &mut Vec<(&str, *mut crate::ffi::PyObject)>) { + items.sort_unstable_by(|a, b| a.0.cmp(b.0)); +} + +pub(crate) struct DictNonStrKey { + dict: PyDictRef, + state: SerializerState, + default: Option>, +} + +impl DictNonStrKey { + fn pyobject_to_string( + key: *mut crate::ffi::PyObject, + opts: crate::opt::Opt, + ) -> Result { + unsafe { + match pyobject_to_obtype(key, opts) { + ObType::None => Ok(String::from("null")), + ObType::Bool => { + if unsafe { core::ptr::eq(key, TRUE) } { + Ok(String::from("true")) + } else { + Ok(String::from("false")) + } + } + ObType::Int => non_str_int(key), + ObType::Float => non_str_float(key), + ObType::Datetime => non_str_datetime(PyDateTimeRef::from_ptr_unchecked(key), opts), + ObType::Date => non_str_date(PyDateRef::from_ptr_unchecked(key)), + ObType::Time => non_str_time(PyTimeRef::from_ptr_unchecked(key), opts), + ObType::Uuid => non_str_uuid(PyUuidRef::from_ptr_unchecked(key)), + ObType::Enum => { + let value = ffi!(PyObject_GetAttr(key, VALUE_STR)); + debug_assert!(ffi!(Py_REFCNT(value)) >= 2); + let ret = Self::pyobject_to_string(value, opts); + ffi!(Py_DECREF(value)); + ret + } + ObType::Str => non_str_str(PyStrRef::from_ptr_unchecked(key)), + ObType::StrSubclass => { + non_str_str_subclass(PyStrSubclassRef::from_ptr_unchecked(key)) + } + ObType::Tuple + | ObType::NumpyScalar + | ObType::NumpyArray + | ObType::Dict + | ObType::List + | ObType::Dataclass + | ObType::Fragment + | ObType::Unknown => Err(SerializeError::DictKeyInvalidType), + } + } + } +} + +impl Serialize for DictNonStrKey { + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut pos = 0; + let mut next_key: *mut crate::ffi::PyObject = core::ptr::null_mut(); + let mut next_value: *mut crate::ffi::PyObject = core::ptr::null_mut(); + + unsafe { + crate::ffi::PyDict_Next( + self.dict.as_ptr(), + &raw mut pos, + &raw mut next_key, + &raw mut next_value, + ); + } + + let opts = self.state.opts() & NOT_PASSTHROUGH; + + let len = self.dict.len(); + assume!(len > 0); + + let mut items: Vec<(String, *mut crate::ffi::PyObject)> = Vec::with_capacity(len); + + for _ in 0..len { + let key = next_key; + let value = next_value; + + unsafe { + crate::ffi::PyDict_Next( + self.dict.as_ptr(), + &raw mut pos, + &raw mut next_key, + &raw mut next_value, + ); + } + + match PyStrRef::from_ptr(key) { + Ok(pystr) => match pystr.as_str() { + Some(uni) => { + items.push((String::from(uni), value)); + } + None => err!(SerializeError::InvalidStr), + }, + Err(_) => match Self::pyobject_to_string(key, opts) { + Ok(key_as_str) => items.push((key_as_str, value)), + Err(err) => err!(err), + }, + } + } + + let mut items_as_str: Vec<(&str, *mut crate::ffi::PyObject)> = Vec::with_capacity(len); + items + .iter() + .for_each(|(key, val)| items_as_str.push(((*key).as_str(), *val))); + + if opt_enabled!(opts, SORT_KEYS) { + sort_dict_items(&mut items_as_str); + } + + let mut map = serializer.serialize_map(None).unwrap(); + for (key, val) in items_as_str.iter() { + let pyvalue = PyObjectSerializer::new(*val, self.state, self.default); + map.serialize_key(key).unwrap(); + map.serialize_value(&pyvalue)?; + } + map.end() + } +} diff --git a/src/serialize/per_type/float.rs b/src/serialize/per_type/float.rs new file mode 100644 index 00000000..89eb4849 --- /dev/null +++ b/src/serialize/per_type/float.rs @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2018-2026) + +use crate::ffi::PyFloatRef; +use serde::ser::{Serialize, Serializer}; + +#[repr(transparent)] +pub(crate) struct FloatSerializer { + ob: PyFloatRef, +} + +impl FloatSerializer { + pub fn new(ptr: PyFloatRef) -> Self { + FloatSerializer { ob: ptr } + } +} + +impl Serialize for FloatSerializer { + #[inline(always)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_f64(self.ob.value()) + } +} diff --git a/src/serialize/per_type/fragment.rs b/src/serialize/per_type/fragment.rs new file mode 100644 index 00000000..2695e664 --- /dev/null +++ b/src/serialize/per_type/fragment.rs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2018-2026) + +use crate::ffi::{PyFragmentRef, PyFragmentRefError}; +use crate::serialize::error::SerializeError; + +use serde::ser::{Serialize, Serializer}; + +#[repr(transparent)] +pub(crate) struct FragmentSerializer { + ob: PyFragmentRef, +} + +impl FragmentSerializer { + pub fn new(ob: PyFragmentRef) -> Self { + FragmentSerializer { ob: ob } + } +} + +impl Serialize for FragmentSerializer { + #[cold] + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self.ob.value() { + Ok(buffer) => serializer.serialize_bytes(buffer), + Err(PyFragmentRefError::InvalidStr) => err!(SerializeError::InvalidStr), + Err(PyFragmentRefError::InvalidFragment) => err!(SerializeError::InvalidFragment), + } + } +} diff --git a/src/serialize/per_type/int.rs b/src/serialize/per_type/int.rs new file mode 100644 index 00000000..26d20761 --- /dev/null +++ b/src/serialize/per_type/int.rs @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2018-2026) + +use crate::ffi::PyIntRef; +use crate::opt::{Opt, STRICT_INTEGER}; +use crate::serialize::error::SerializeError; +use serde::ser::{Serialize, Serializer}; + +// https://tools.ietf.org/html/rfc7159#section-6 +// "[-(2**53)+1, (2**53)-1]" +const STRICT_INT_MIN: i64 = -9007199254740991; +const STRICT_INT_MAX: i64 = 9007199254740991; + +pub(crate) struct IntSerializer { + ob: PyIntRef, + opts: Opt, +} + +impl IntSerializer { + pub fn new(ob: PyIntRef, opts: Opt) -> Self { + IntSerializer { ob: ob, opts: opts } + } +} + +impl Serialize for IntSerializer { + #[inline(always)] + #[cfg(feature = "inline_int")] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + unsafe { + match self.ob.kind() { + crate::ffi::PyIntKind::I32 => serializer.serialize_i32(self.ob.as_i32()), + crate::ffi::PyIntKind::U32 => serializer.serialize_u32(self.ob.as_u32()), + crate::ffi::PyIntKind::I64 => { + let value = self + .ob + .as_i64() + .map_err(|_| serde::ser::Error::custom(SerializeError::Integer64Bits))?; + if opt_enabled!(self.opts, STRICT_INTEGER) + && !(STRICT_INT_MIN..=STRICT_INT_MAX).contains(&value) + { + cold_path!(); + err!(SerializeError::Integer53Bits); + } + serializer.serialize_i64(value) + } + crate::ffi::PyIntKind::U64 => { + let value = self + .ob + .as_u64() + .map_err(|_| serde::ser::Error::custom(SerializeError::Integer64Bits))?; + if opt_enabled!(self.opts, STRICT_INTEGER) && value > STRICT_INT_MAX as u64 { + cold_path!(); + err!(SerializeError::Integer53Bits); + } + serializer.serialize_u64(value) + } + } + } + } + + #[inline(always)] + #[cfg(not(feature = "inline_int"))] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + unsafe { + match self.ob.as_i64() { + Ok(value) => { + if opt_enabled!(self.opts, STRICT_INTEGER) + && !(STRICT_INT_MIN..=STRICT_INT_MAX).contains(&value) + { + cold_path!(); + err!(SerializeError::Integer53Bits); + } + serializer.serialize_i64(value) + } + Err(_) => match self.ob.as_u64() { + Ok(value) => { + if opt_enabled!(self.opts, STRICT_INTEGER) && value > STRICT_INT_MAX as u64 + { + cold_path!(); + err!(SerializeError::Integer53Bits); + } + serializer.serialize_u64(value) + } + Err(_) => err!(SerializeError::Integer64Bits), + }, + } + } + } +} diff --git a/src/serialize/per_type/list.rs b/src/serialize/per_type/list.rs new file mode 100644 index 00000000..8003ee9f --- /dev/null +++ b/src/serialize/per_type/list.rs @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2018-2026) + +use crate::ffi::{ + PyBoolRef, PyDateRef, PyDateTimeRef, PyDictRef, PyFloatRef, PyFragmentRef, PyIntRef, PyListRef, + PyStrRef, PyStrSubclassRef, PyTimeRef, PyUuidRef, +}; +use crate::serialize::error::SerializeError; +use crate::serialize::numpy::NumpyScalar; +use crate::serialize::obtype::{ObType, pyobject_to_obtype}; +use crate::serialize::per_type::{ + BoolSerializer, DataclassGenericSerializer, Date, DateTime, DefaultSerializer, + DictGenericSerializer, EnumSerializer, FloatSerializer, FragmentSerializer, IntSerializer, + NoneSerializer, NumpySerializer, StrSerializer, StrSubclassSerializer, Time, UUID, +}; +use crate::serialize::serializer::PyObjectSerializer; +use crate::serialize::state::SerializerState; +use crate::typeref::TUPLE_TYPE; +use crate::util::isize_to_usize; + +use core::ptr::NonNull; +use serde::ser::{Serialize, SerializeSeq, Serializer}; + +pub(crate) struct ZeroListSerializer; + +impl ZeroListSerializer { + pub const fn new() -> Self { + Self {} + } +} + +impl Serialize for ZeroListSerializer { + #[inline(always)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_bytes(b"[]") + } +} + +pub(crate) struct ListTupleSerializer { + data_ptr: *const *mut crate::ffi::PyObject, + state: SerializerState, + default: Option>, + len: usize, +} + +impl ListTupleSerializer { + pub fn from_list( + ob: PyListRef, + state: SerializerState, + default: Option>, + ) -> Self { + Self { + data_ptr: ob.data_ptr(), + len: ob.len(), + state: state.copy_for_recursive_call(), + default: default, + } + } + + pub fn from_tuple( + ptr: *mut crate::ffi::PyObject, + state: SerializerState, + default: Option>, + ) -> Self { + debug_assert!( + is_type!(crate::ffi::PyObject_Type(ptr), TUPLE_TYPE) + || is_subclass_by_flag!( + crate::ffi::PyType_GetFlags(crate::ffi::PyObject_Type(ptr)), + Py_TPFLAGS_TUPLE_SUBCLASS + ) + ); + let data_ptr = unsafe { (*ptr.cast::()).ob_item.as_ptr() }; + let len = isize_to_usize(ffi!(Py_SIZE(ptr))); + Self { + data_ptr: data_ptr, + len: len, + state: state.copy_for_recursive_call(), + default: default, + } + } +} + +impl Serialize for ListTupleSerializer { + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + if self.state.recursion_limit() { + cold_path!(); + err!(SerializeError::RecursionLimit) + } + debug_assert!(self.len >= 1); + let mut seq = serializer.serialize_seq(None).unwrap(); + for idx in 0..self.len { + let value = unsafe { *((self.data_ptr).add(idx)) }; + match pyobject_to_obtype(value, self.state.opts()) { + ObType::Str => { + seq.serialize_element(&StrSerializer::new(unsafe { + PyStrRef::from_ptr_unchecked(value) + }))?; + } + ObType::StrSubclass => { + seq.serialize_element(&StrSubclassSerializer::new(unsafe { + PyStrSubclassRef::from_ptr_unchecked(value) + }))?; + } + ObType::Int => { + seq.serialize_element(&IntSerializer::new( + unsafe { PyIntRef::from_ptr_unchecked(value) }, + self.state.opts(), + ))?; + } + ObType::None => { + seq.serialize_element(&NoneSerializer::new()).unwrap(); + } + ObType::Float => { + seq.serialize_element(&FloatSerializer::new(unsafe { + PyFloatRef::from_ptr_unchecked(value) + }))?; + } + ObType::Bool => { + seq.serialize_element(&BoolSerializer::new(unsafe { + PyBoolRef::from_ptr_unchecked(value) + })) + .unwrap(); + } + ObType::Datetime => { + seq.serialize_element(&DateTime::new( + unsafe { PyDateTimeRef::from_ptr_unchecked(value) }, + self.state.opts(), + ))?; + } + ObType::Date => { + seq.serialize_element(&Date::new(unsafe { + PyDateRef::from_ptr_unchecked(value) + }))?; + } + ObType::Time => { + seq.serialize_element(&Time::new( + unsafe { PyTimeRef::from_ptr_unchecked(value) }, + self.state.opts(), + ))?; + } + ObType::Uuid => { + seq.serialize_element(&UUID::new(unsafe { + PyUuidRef::from_ptr_unchecked(value) + })) + .unwrap(); + } + ObType::Dict => { + let pyvalue = DictGenericSerializer::new( + unsafe { PyDictRef::from_ptr_unchecked(value) }, + self.state, + self.default, + ); + seq.serialize_element(&pyvalue)?; + } + ObType::List => { + if ffi!(Py_SIZE(value)) == 0 { + seq.serialize_element(&ZeroListSerializer::new()).unwrap(); + } else { + let pyvalue = ListTupleSerializer::from_list( + unsafe { PyListRef::from_ptr_unchecked(value) }, + self.state, + self.default, + ); + seq.serialize_element(&pyvalue)?; + } + } + ObType::Tuple => { + if ffi!(Py_SIZE(value)) == 0 { + seq.serialize_element(&ZeroListSerializer::new()).unwrap(); + } else { + let pyvalue = + ListTupleSerializer::from_tuple(value, self.state, self.default); + seq.serialize_element(&pyvalue)?; + } + } + ObType::Dataclass => { + seq.serialize_element(&DataclassGenericSerializer::new( + &PyObjectSerializer::new(value, self.state, self.default), + ))?; + } + ObType::Enum => { + seq.serialize_element(&EnumSerializer::new(&PyObjectSerializer::new( + value, + self.state, + self.default, + )))?; + } + ObType::NumpyArray => { + seq.serialize_element(&NumpySerializer::new(&PyObjectSerializer::new( + value, + self.state, + self.default, + )))?; + } + ObType::NumpyScalar => { + seq.serialize_element(&NumpyScalar::new(value, self.state.opts()))?; + } + ObType::Fragment => { + seq.serialize_element(&FragmentSerializer::new(unsafe { + PyFragmentRef::from_ptr_unchecked(value) + }))?; + } + ObType::Unknown => { + seq.serialize_element(&DefaultSerializer::new(&PyObjectSerializer::new( + value, + self.state, + self.default, + )))?; + } + } + } + seq.end() + } +} diff --git a/src/serialize/per_type/mod.rs b/src/serialize/per_type/mod.rs new file mode 100644 index 00000000..91016fb3 --- /dev/null +++ b/src/serialize/per_type/mod.rs @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2023-2026) + +mod dataclass; +mod datetime; +mod default; +mod dict; +mod float; +mod fragment; +mod int; +mod list; +mod none; +mod numpy; +mod pybool; +mod pyenum; +mod unicode; +mod uuid; + +pub(crate) use dataclass::DataclassGenericSerializer; +pub(crate) use datetime::{Date, DateTime, Time}; +pub(crate) use default::DefaultSerializer; +pub(crate) use dict::DictGenericSerializer; +pub(crate) use float::FloatSerializer; +pub(crate) use fragment::FragmentSerializer; +pub(crate) use int::IntSerializer; +pub(crate) use list::{ListTupleSerializer, ZeroListSerializer}; +pub(crate) use none::NoneSerializer; +pub(crate) use numpy::{NumpySerializer, is_numpy_array, is_numpy_scalar}; +pub(crate) use pybool::BoolSerializer; +pub(crate) use pyenum::EnumSerializer; +pub(crate) use unicode::{StrSerializer, StrSubclassSerializer}; +pub(crate) use uuid::UUID; diff --git a/src/serialize/per_type/none.rs b/src/serialize/per_type/none.rs new file mode 100644 index 00000000..830daf95 --- /dev/null +++ b/src/serialize/per_type/none.rs @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2018-2025) + +use serde::ser::{Serialize, Serializer}; + +pub(crate) struct NoneSerializer; + +impl NoneSerializer { + pub const fn new() -> Self { + Self {} + } +} + +impl Serialize for NoneSerializer { + #[inline] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_unit() + } +} diff --git a/src/serialize/per_type/numpy.rs b/src/serialize/per_type/numpy.rs new file mode 100644 index 00000000..2e052690 --- /dev/null +++ b/src/serialize/per_type/numpy.rs @@ -0,0 +1,738 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright ijl (2018-2026), Nazar Kostetskyi (2022), Aviram Hassan (2020-2021), Ben Sully (2021) + +use crate::ffi::{ + NumpyBool, NumpyDatetime64, NumpyDatetime64Repr, NumpyDatetimeUnit, NumpyFloat16, NumpyFloat32, + NumpyFloat64, NumpyInt8, NumpyInt16, NumpyInt32, NumpyInt64, NumpyUint8, NumpyUint16, + NumpyUint32, NumpyUint64, PyTypeObject, +}; +use crate::serialize::error::SerializeError; +use crate::serialize::numpy::{ + ItemType, NumpyArray, NumpyBoolArray, NumpyDatetime64Array, NumpyF16Array, NumpyF32Array, + NumpyF64Array, NumpyI8Array, NumpyI16Array, NumpyI32Array, NumpyI64Array, NumpyScalar, + NumpyU8Array, NumpyU16Array, NumpyU32Array, NumpyU64Array, PyArrayError, datetime_into_error, + write_numpy_datetime, +}; +use crate::serialize::per_type::{DefaultSerializer, ZeroListSerializer}; +use crate::serialize::serializer::PyObjectSerializer; +use crate::serialize::writer::SmallFixedBuffer; +use crate::serialize::writer::f16_to_f32; +use crate::typeref::{NUMPY_TYPES, load_numpy_types}; +use serde::ser::{Serialize, SerializeSeq, Serializer}; + +#[repr(transparent)] +pub(crate) struct NumpySerializer<'a> { + previous: &'a PyObjectSerializer, +} + +impl<'a> NumpySerializer<'a> { + pub fn new(previous: &'a PyObjectSerializer) -> Self { + Self { previous: previous } + } +} + +impl Serialize for NumpySerializer<'_> { + #[cold] + #[inline(never)] + #[cfg_attr(feature = "optimize", optimize(size))] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match NumpyArray::new(self.previous.ptr, self.previous.state.opts()) { + Ok(val) => val.serialize(serializer), + Err(PyArrayError::Malformed) => err!(SerializeError::NumpyMalformed), + Err(PyArrayError::NotContiguous | PyArrayError::UnsupportedDataType) + if self.previous.default.is_some() => + { + DefaultSerializer::new(self.previous).serialize(serializer) + } + Err(PyArrayError::NotContiguous) => { + err!(SerializeError::NumpyNotCContiguous) + } + Err(PyArrayError::NotNativeEndian) => { + err!(SerializeError::NumpyNotNativeEndian) + } + Err(PyArrayError::UnsupportedDataType) => { + err!(SerializeError::NumpyUnsupportedDatatype) + } + } + } +} + +macro_rules! slice { + ($ptr:expr, $size:expr) => { + unsafe { core::slice::from_raw_parts($ptr, $size) } + }; +} + +#[cold] +pub(crate) fn is_numpy_scalar(ob_type: *mut PyTypeObject) -> bool { + let numpy_types = unsafe { NUMPY_TYPES.get_or_init(load_numpy_types) }; + if numpy_types.is_none() { + false + } else { + let scalar_types = unsafe { numpy_types.unwrap().as_ref() }; + core::ptr::eq(ob_type, scalar_types.float64) + || core::ptr::eq(ob_type, scalar_types.float32) + || core::ptr::eq(ob_type, scalar_types.float16) + || core::ptr::eq(ob_type, scalar_types.int64) + || core::ptr::eq(ob_type, scalar_types.int16) + || core::ptr::eq(ob_type, scalar_types.int32) + || core::ptr::eq(ob_type, scalar_types.int8) + || core::ptr::eq(ob_type, scalar_types.uint64) + || core::ptr::eq(ob_type, scalar_types.uint32) + || core::ptr::eq(ob_type, scalar_types.uint8) + || core::ptr::eq(ob_type, scalar_types.uint16) + || core::ptr::eq(ob_type, scalar_types.bool_) + || core::ptr::eq(ob_type, scalar_types.datetime64) + } +} + +#[cold] +pub(crate) fn is_numpy_array(ob_type: *mut PyTypeObject) -> bool { + let numpy_types = unsafe { NUMPY_TYPES.get_or_init(load_numpy_types) }; + if numpy_types.is_none() { + false + } else { + let scalar_types = unsafe { numpy_types.unwrap().as_ref() }; + unsafe { core::ptr::eq(ob_type, scalar_types.array) } + } +} + +impl Serialize for NumpyArray { + #[cold] + #[inline(never)] + #[cfg_attr(feature = "optimize", optimize(size))] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + if !(self.depth >= self.dimensions() || self.shape()[self.depth] != 0) { + cold_path!(); + ZeroListSerializer::new().serialize(serializer) + } else if !self.children.is_empty() { + cold_path!(); + let mut seq = serializer.serialize_seq(None).unwrap(); + for child in &self.children { + seq.serialize_element(child).unwrap(); + } + seq.end() + } else { + match self.kind { + ItemType::F64 => { + NumpyF64Array::new(slice!(self.data().cast::(), self.num_items())) + .serialize(serializer) + } + ItemType::F32 => { + NumpyF32Array::new(slice!(self.data().cast::(), self.num_items())) + .serialize(serializer) + } + ItemType::F16 => { + NumpyF16Array::new(slice!(self.data().cast::(), self.num_items())) + .serialize(serializer) + } + ItemType::U64 => { + NumpyU64Array::new(slice!(self.data().cast::(), self.num_items())) + .serialize(serializer) + } + ItemType::U32 => { + NumpyU32Array::new(slice!(self.data().cast::(), self.num_items())) + .serialize(serializer) + } + ItemType::U16 => { + NumpyU16Array::new(slice!(self.data().cast::(), self.num_items())) + .serialize(serializer) + } + ItemType::U8 => { + NumpyU8Array::new(slice!(self.data().cast::(), self.num_items())) + .serialize(serializer) + } + ItemType::I64 => { + NumpyI64Array::new(slice!(self.data().cast::(), self.num_items())) + .serialize(serializer) + } + ItemType::I32 => { + NumpyI32Array::new(slice!(self.data().cast::(), self.num_items())) + .serialize(serializer) + } + ItemType::I16 => { + NumpyI16Array::new(slice!(self.data().cast::(), self.num_items())) + .serialize(serializer) + } + ItemType::I8 => { + NumpyI8Array::new(slice!(self.data().cast::(), self.num_items())) + .serialize(serializer) + } + ItemType::BOOL => { + NumpyBoolArray::new(slice!(self.data().cast::(), self.num_items())) + .serialize(serializer) + } + ItemType::DATETIME64(unit) => NumpyDatetime64Array::new( + slice!(self.data().cast::(), self.num_items()), + unit, + self.opts, + ) + .serialize(serializer), + } + } + } +} + +impl Serialize for NumpyF64Array<'_> { + #[cold] + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(None).unwrap(); + for &each in self.data.iter() { + seq.serialize_element(&DataTypeF64 { obj: each }).unwrap(); + } + seq.end() + } +} + +#[repr(transparent)] +pub(crate) struct DataTypeF64 { + obj: f64, +} + +impl Serialize for DataTypeF64 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_f64(self.obj) + } +} + +impl Serialize for NumpyF32Array<'_> { + #[cold] + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(None).unwrap(); + for &each in self.data.iter() { + seq.serialize_element(&DataTypeF32 { obj: each }).unwrap(); + } + seq.end() + } +} + +#[repr(transparent)] +struct DataTypeF32 { + obj: f32, +} + +impl Serialize for DataTypeF32 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_f32(self.obj) + } +} + +impl Serialize for NumpyF16Array<'_> { + #[cold] + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(None).unwrap(); + for &each in self.data.iter() { + seq.serialize_element(&DataTypeF16 { obj: each }).unwrap(); + } + seq.end() + } +} + +#[repr(transparent)] +struct DataTypeF16 { + obj: u16, +} + +impl Serialize for DataTypeF16 { + #[cold] + #[cfg_attr(feature = "optimize", optimize(size))] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_f32(f16_to_f32(self.obj)) + } +} + +impl Serialize for NumpyU64Array<'_> { + #[cold] + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(None).unwrap(); + for &each in self.data.iter() { + seq.serialize_element(&DataTypeU64 { obj: each }).unwrap(); + } + seq.end() + } +} + +#[repr(transparent)] +pub(crate) struct DataTypeU64 { + obj: u64, +} + +impl Serialize for DataTypeU64 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_u64(self.obj) + } +} + +impl Serialize for NumpyU32Array<'_> { + #[cold] + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(None).unwrap(); + for &each in self.data.iter() { + seq.serialize_element(&DataTypeU32 { obj: each }).unwrap(); + } + seq.end() + } +} + +#[repr(transparent)] +pub(crate) struct DataTypeU32 { + obj: u32, +} + +impl Serialize for DataTypeU32 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_u32(self.obj) + } +} + +impl Serialize for NumpyU16Array<'_> { + #[cold] + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(None).unwrap(); + for &each in self.data.iter() { + seq.serialize_element(&DataTypeU16 { obj: each }).unwrap(); + } + seq.end() + } +} + +#[repr(transparent)] +pub(crate) struct DataTypeU16 { + obj: u16, +} + +impl Serialize for DataTypeU16 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_u32(u32::from(self.obj)) + } +} + +impl Serialize for NumpyI64Array<'_> { + #[cold] + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(None).unwrap(); + for &each in self.data.iter() { + seq.serialize_element(&DataTypeI64 { obj: each }).unwrap(); + } + seq.end() + } +} + +#[repr(transparent)] +pub(crate) struct DataTypeI64 { + obj: i64, +} + +impl Serialize for DataTypeI64 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_i64(self.obj) + } +} + +impl Serialize for NumpyI32Array<'_> { + #[cold] + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(None).unwrap(); + for &each in self.data.iter() { + seq.serialize_element(&DataTypeI32 { obj: each }).unwrap(); + } + seq.end() + } +} + +#[repr(transparent)] +pub(crate) struct DataTypeI32 { + obj: i32, +} + +impl Serialize for DataTypeI32 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_i32(self.obj) + } +} + +impl Serialize for NumpyI16Array<'_> { + #[cold] + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(None).unwrap(); + for &each in self.data.iter() { + seq.serialize_element(&DataTypeI16 { obj: each }).unwrap(); + } + seq.end() + } +} + +#[repr(transparent)] +pub(crate) struct DataTypeI16 { + obj: i16, +} + +impl Serialize for DataTypeI16 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_i32(i32::from(self.obj)) + } +} + +impl Serialize for NumpyI8Array<'_> { + #[cold] + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(None).unwrap(); + for &each in self.data.iter() { + seq.serialize_element(&DataTypeI8 { obj: each }).unwrap(); + } + seq.end() + } +} + +#[repr(transparent)] +pub(crate) struct DataTypeI8 { + obj: i8, +} + +impl Serialize for DataTypeI8 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_i32(i32::from(self.obj)) + } +} + +impl Serialize for NumpyU8Array<'_> { + #[cold] + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(None).unwrap(); + for &each in self.data.iter() { + seq.serialize_element(&DataTypeU8 { obj: each }).unwrap(); + } + seq.end() + } +} + +#[repr(transparent)] +pub(crate) struct DataTypeU8 { + obj: u8, +} + +impl Serialize for DataTypeU8 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_u32(u32::from(self.obj)) + } +} + +impl Serialize for NumpyBoolArray<'_> { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(None).unwrap(); + for &each in self.data.iter() { + seq.serialize_element(&DataTypeBool { obj: each }).unwrap(); + } + seq.end() + } +} + +#[repr(transparent)] +pub(crate) struct DataTypeBool { + obj: u8, +} + +impl Serialize for DataTypeBool { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_bool(self.obj == 1) + } +} + +impl Serialize for NumpyScalar { + #[cold] + #[inline(never)] + #[cfg_attr(feature = "optimize", optimize(size))] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + unsafe { + let ob_type = crate::ffi::PyObject_Type(self.ptr); + let scalar_types = + unsafe { NUMPY_TYPES.get_or_init(load_numpy_types).unwrap().as_ref() }; + if core::ptr::eq(ob_type, scalar_types.float64) { + (*(self.ptr.cast::())).serialize(serializer) + } else if core::ptr::eq(ob_type, scalar_types.float32) { + (*(self.ptr.cast::())).serialize(serializer) + } else if core::ptr::eq(ob_type, scalar_types.float16) { + (*(self.ptr.cast::())).serialize(serializer) + } else if core::ptr::eq(ob_type, scalar_types.int64) { + (*(self.ptr.cast::())).serialize(serializer) + } else if core::ptr::eq(ob_type, scalar_types.int32) { + (*(self.ptr.cast::())).serialize(serializer) + } else if core::ptr::eq(ob_type, scalar_types.int16) { + (*(self.ptr.cast::())).serialize(serializer) + } else if core::ptr::eq(ob_type, scalar_types.int8) { + (*(self.ptr.cast::())).serialize(serializer) + } else if core::ptr::eq(ob_type, scalar_types.uint64) { + (*(self.ptr.cast::())).serialize(serializer) + } else if core::ptr::eq(ob_type, scalar_types.uint32) { + (*(self.ptr.cast::())).serialize(serializer) + } else if core::ptr::eq(ob_type, scalar_types.uint16) { + (*(self.ptr.cast::())).serialize(serializer) + } else if core::ptr::eq(ob_type, scalar_types.uint8) { + (*(self.ptr.cast::())).serialize(serializer) + } else if core::ptr::eq(ob_type, scalar_types.bool_) { + (*(self.ptr.cast::())).serialize(serializer) + } else if core::ptr::eq(ob_type, scalar_types.datetime64) { + let unit = NumpyDatetimeUnit::from_pyobject(self.ptr); + let obj = &*self.ptr.cast::(); + let dt = unit + .datetime(obj.value, self.opts) + .map_err(|e| serde::ser::Error::custom(datetime_into_error(e)))?; + dt.serialize(serializer) + } else { + unreachable!() + } + } + } +} +impl Serialize for NumpyInt8 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_i32(i32::from(self.value)) + } +} + +impl Serialize for NumpyInt16 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_i32(i32::from(self.value)) + } +} + +impl Serialize for NumpyInt32 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_i32(self.value) + } +} + +impl Serialize for NumpyInt64 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_i64(self.value) + } +} + +impl Serialize for NumpyUint8 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_u32(u32::from(self.value)) + } +} + +impl Serialize for NumpyUint16 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_u32(u32::from(self.value)) + } +} +impl Serialize for NumpyUint32 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_u32(self.value) + } +} + +impl Serialize for NumpyUint64 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_u64(self.value) + } +} + +impl Serialize for NumpyFloat16 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_f32(f16_to_f32(self.value)) + } +} + +impl Serialize for NumpyFloat32 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_f32(self.value) + } +} + +impl Serialize for NumpyFloat64 { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_f64(self.value) + } +} + +impl Serialize for NumpyBool { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_bool(self.value) + } +} + +impl Serialize for NumpyDatetime64Array<'_> { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(None).unwrap(); + for &each in self.data.iter() { + let dt = self + .unit + .datetime(each, self.opts) + .map_err(|e| serde::ser::Error::custom(datetime_into_error(e)))?; + seq.serialize_element(&dt).unwrap(); + } + seq.end() + } +} + +impl Serialize for NumpyDatetime64Repr { + #[cold] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut buf = SmallFixedBuffer::new(); + write_numpy_datetime(self, &mut buf); + serializer.collect_str(str_from_slice!(buf.as_ptr(), buf.len())) + } +} diff --git a/src/serialize/per_type/pybool.rs b/src/serialize/per_type/pybool.rs new file mode 100644 index 00000000..30ffa681 --- /dev/null +++ b/src/serialize/per_type/pybool.rs @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2018-2026) + +use crate::ffi::PyBoolRef; +use serde::ser::{Serialize, Serializer}; + +#[repr(transparent)] +pub(crate) struct BoolSerializer { + ob: PyBoolRef, +} + +impl BoolSerializer { + pub fn new(ob: PyBoolRef) -> Self { + BoolSerializer { ob: ob } + } +} + +impl Serialize for BoolSerializer { + #[inline] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_bool(unsafe { core::ptr::eq(self.ob.as_ptr(), crate::typeref::TRUE) }) + } +} diff --git a/src/serialize/per_type/pyenum.rs b/src/serialize/per_type/pyenum.rs new file mode 100644 index 00000000..7ced8b6d --- /dev/null +++ b/src/serialize/per_type/pyenum.rs @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2018-2025) + +use crate::serialize::serializer::PyObjectSerializer; +use crate::typeref::VALUE_STR; +use serde::ser::{Serialize, Serializer}; + +#[repr(transparent)] +pub(crate) struct EnumSerializer<'a> { + previous: &'a PyObjectSerializer, +} + +impl<'a> EnumSerializer<'a> { + pub fn new(previous: &'a PyObjectSerializer) -> Self { + Self { previous: previous } + } +} + +impl Serialize for EnumSerializer<'_> { + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let value = ffi!(PyObject_GetAttr(self.previous.ptr, VALUE_STR)); + debug_assert!(ffi!(Py_REFCNT(value)) >= 2); + let ret = PyObjectSerializer::new(value, self.previous.state, self.previous.default) + .serialize(serializer); + ffi!(Py_DECREF(value)); + ret + } +} diff --git a/src/serialize/per_type/unicode.rs b/src/serialize/per_type/unicode.rs new file mode 100644 index 00000000..3e490c44 --- /dev/null +++ b/src/serialize/per_type/unicode.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2018-2026) + +use crate::ffi::{PyStrRef, PyStrSubclassRef}; +use crate::serialize::error::SerializeError; + +use serde::ser::{Serialize, Serializer}; + +#[repr(transparent)] +pub(crate) struct StrSerializer { + ob: PyStrRef, +} + +impl StrSerializer { + pub fn new(ptr: PyStrRef) -> Self { + StrSerializer { ob: ptr } + } +} + +impl Serialize for StrSerializer { + #[inline(always)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self.ob.clone().as_str() { + Some(uni) => serializer.serialize_str(uni), + None => { + cold_path!(); + err!(SerializeError::InvalidStr) + } + } + } +} + +#[repr(transparent)] +pub(crate) struct StrSubclassSerializer { + ob: PyStrSubclassRef, +} + +impl StrSubclassSerializer { + pub fn new(ptr: PyStrSubclassRef) -> Self { + StrSubclassSerializer { ob: ptr } + } +} + +impl Serialize for StrSubclassSerializer { + #[inline(never)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self.ob.as_str() { + Some(uni) => serializer.serialize_str(uni), + None => { + cold_path!(); + err!(SerializeError::InvalidStr) + } + } + } +} diff --git a/src/serialize/per_type/uuid.rs b/src/serialize/per_type/uuid.rs new file mode 100644 index 00000000..76229344 --- /dev/null +++ b/src/serialize/per_type/uuid.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2018-2026) + +use crate::ffi::PyUuidRef; +use crate::serialize::uuid::write_uuid; +use crate::serialize::writer::SmallFixedBuffer; +use serde::ser::{Serialize, Serializer}; + +#[repr(transparent)] +pub(crate) struct UUID { + ob: PyUuidRef, +} + +impl UUID { + pub fn new(ptr: PyUuidRef) -> Self { + UUID { ob: ptr } + } +} + +impl Serialize for UUID { + #[inline(always)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut buf = SmallFixedBuffer::new(); + write_uuid(self.ob.clone(), &mut buf); + serializer.serialize_unit_struct(str_from_slice!(buf.as_ptr(), buf.len())) + } +} diff --git a/src/serialize/serializer.rs b/src/serialize/serializer.rs index ada89139..63f207d8 100644 --- a/src/serialize/serializer.rs +++ b/src/serialize/serializer.rs @@ -1,174 +1,60 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2018-2026) -use crate::ffi::{PyDict_GET_SIZE, PyTypeObject}; -use crate::opt::*; -use crate::serialize::dataclass::*; -use crate::serialize::datetime::*; -use crate::serialize::default::*; -use crate::serialize::dict::*; -use crate::serialize::error::*; -use crate::serialize::int::*; -use crate::serialize::list::*; -use crate::serialize::numpy::*; -use crate::serialize::str::*; -use crate::serialize::tuple::*; -use crate::serialize::uuid::*; -use crate::serialize::writer::*; -use crate::typeref::*; -use serde::ser::{Serialize, SerializeMap, SerializeSeq, Serializer}; -use std::io::Write; -use std::ptr::NonNull; +use crate::ffi::{ + PyBoolRef, PyDateRef, PyDateTimeRef, PyDictRef, PyFloatRef, PyFragmentRef, PyIntRef, PyListRef, + PyStrRef, PyStrSubclassRef, PyTimeRef, PyUuidRef, +}; +use crate::opt::{APPEND_NEWLINE, INDENT_2, Opt}; +use crate::serialize::numpy::NumpyScalar; +use crate::serialize::obtype::{ObType, pyobject_to_obtype}; +use crate::serialize::per_type::{ + BoolSerializer, DataclassGenericSerializer, Date, DateTime, DefaultSerializer, + DictGenericSerializer, EnumSerializer, FloatSerializer, FragmentSerializer, IntSerializer, + ListTupleSerializer, NoneSerializer, NumpySerializer, StrSerializer, StrSubclassSerializer, + Time, UUID, ZeroListSerializer, +}; +use crate::serialize::state::SerializerState; +use crate::serialize::writer::{BytesWriter, to_writer, to_writer_pretty}; +use core::ptr::NonNull; +use serde::ser::{Serialize, Serializer}; -pub const RECURSION_LIMIT: u8 = 255; - -pub fn serialize( - ptr: *mut pyo3_ffi::PyObject, - default: Option>, +pub(crate) fn serialize( + ptr: *mut crate::ffi::PyObject, + default: Option>, opts: Opt, -) -> Result, String> { +) -> Result, String> { let mut buf = BytesWriter::default(); - let obj = PyObjectSerializer::new(ptr, opts, 0, 0, default); - let res = if opts & INDENT_2 != INDENT_2 { - serde_json::to_writer(&mut buf, &obj) + let obj = PyObjectSerializer::new(ptr, SerializerState::new(opts), default); + let res = if opt_disabled!(opts, INDENT_2) { + to_writer(&mut buf, &obj) } else { - serde_json::to_writer_pretty(&mut buf, &obj) + to_writer_pretty(&mut buf, &obj) }; match res { - Ok(_) => { - if opts & APPEND_NEWLINE != 0 { - let _ = buf.write(b"\n"); - } - Ok(buf.finish()) - } + Ok(()) => Ok(buf.finish(opt_enabled!(opts, APPEND_NEWLINE))), Err(err) => { - ffi!(_Py_Dealloc(buf.finish().as_ptr())); + buf.abort(); Err(err.to_string()) } } } -#[derive(Copy, Clone)] -pub enum ObType { - Str, - Int, - Bool, - None, - Float, - List, - Dict, - Datetime, - Date, - Time, - Tuple, - Uuid, - Dataclass, - NumpyScalar, - NumpyArray, - Enum, - StrSubclass, - Unknown, -} - -pub fn pyobject_to_obtype(obj: *mut pyo3_ffi::PyObject, opts: Opt) -> ObType { - unsafe { - let ob_type = ob_type!(obj); - if ob_type == STR_TYPE { - ObType::Str - } else if ob_type == INT_TYPE { - ObType::Int - } else if ob_type == BOOL_TYPE { - ObType::Bool - } else if ob_type == NONE_TYPE { - ObType::None - } else if ob_type == FLOAT_TYPE { - ObType::Float - } else if ob_type == LIST_TYPE { - ObType::List - } else if ob_type == DICT_TYPE { - ObType::Dict - } else if ob_type == DATETIME_TYPE && opts & PASSTHROUGH_DATETIME == 0 { - ObType::Datetime - } else { - pyobject_to_obtype_unlikely(obj, opts) - } - } -} - -macro_rules! is_subclass { - ($ob_type:expr, $flag:ident) => { - (((*$ob_type).tp_flags & pyo3_ffi::$flag) != 0) - }; -} - -#[cold] -#[cfg_attr(feature = "unstable-simd", optimize(size))] -#[inline(never)] -pub fn pyobject_to_obtype_unlikely(obj: *mut pyo3_ffi::PyObject, opts: Opt) -> ObType { - unsafe { - let ob_type = ob_type!(obj); - if ob_type == DATE_TYPE && opts & PASSTHROUGH_DATETIME == 0 { - ObType::Date - } else if ob_type == TIME_TYPE && opts & PASSTHROUGH_DATETIME == 0 { - ObType::Time - } else if ob_type == TUPLE_TYPE { - ObType::Tuple - } else if ob_type == UUID_TYPE { - ObType::Uuid - } else if (*(ob_type as *mut PyTypeObject)).ob_type == ENUM_TYPE { - ObType::Enum - } else if opts & PASSTHROUGH_SUBCLASS == 0 - && is_subclass!(ob_type, Py_TPFLAGS_UNICODE_SUBCLASS) - { - ObType::StrSubclass - } else if opts & PASSTHROUGH_SUBCLASS == 0 - && is_subclass!(ob_type, Py_TPFLAGS_LONG_SUBCLASS) - { - ObType::Int - } else if opts & PASSTHROUGH_SUBCLASS == 0 - && is_subclass!(ob_type, Py_TPFLAGS_LIST_SUBCLASS) - { - ObType::List - } else if opts & PASSTHROUGH_SUBCLASS == 0 - && is_subclass!(ob_type, Py_TPFLAGS_DICT_SUBCLASS) - { - ObType::Dict - } else if opts & PASSTHROUGH_DATACLASS == 0 - && ffi!(PyDict_Contains((*ob_type).tp_dict, DATACLASS_FIELDS_STR)) == 1 - { - ObType::Dataclass - } else if opts & SERIALIZE_NUMPY != 0 && is_numpy_scalar(ob_type) { - ObType::NumpyScalar - } else if opts & SERIALIZE_NUMPY != 0 && is_numpy_array(ob_type) { - ObType::NumpyArray - } else { - ObType::Unknown - } - } -} - -pub struct PyObjectSerializer { - ptr: *mut pyo3_ffi::PyObject, - obtype: ObType, - opts: Opt, - default_calls: u8, - recursion: u8, - default: Option>, +pub(crate) struct PyObjectSerializer { + pub ptr: *mut crate::ffi::PyObject, + pub state: SerializerState, + pub default: Option>, } impl PyObjectSerializer { pub fn new( - ptr: *mut pyo3_ffi::PyObject, - opts: Opt, - default_calls: u8, - recursion: u8, - default: Option>, + ptr: *mut crate::ffi::PyObject, + state: SerializerState, + default: Option>, ) -> Self { PyObjectSerializer { ptr: ptr, - obtype: pyobject_to_obtype(ptr, opts), - opts: opts, - default_calls: default_calls, - recursion: recursion, + state: state, default: default, } } @@ -179,159 +65,78 @@ impl Serialize for PyObjectSerializer { where S: Serializer, { - match self.obtype { - ObType::Str => StrSerializer::new(self.ptr).serialize(serializer), - ObType::StrSubclass => StrSubclassSerializer::new(self.ptr).serialize(serializer), - ObType::Int => { - if unlikely!(self.opts & STRICT_INTEGER != 0) { - Int53Serializer::new(self.ptr).serialize(serializer) - } else { - IntSerializer::new(self.ptr).serialize(serializer) + unsafe { + match pyobject_to_obtype(self.ptr, self.state.opts()) { + ObType::Str => { + StrSerializer::new(PyStrRef::from_ptr_unchecked(self.ptr)).serialize(serializer) } - } - ObType::None => serializer.serialize_unit(), - ObType::Float => serializer.serialize_f64(ffi!(PyFloat_AS_DOUBLE(self.ptr))), - ObType::Bool => serializer.serialize_bool(unsafe { self.ptr == TRUE }), - ObType::Datetime => DateTime::new(self.ptr, self.opts).serialize(serializer), - ObType::Date => Date::new(self.ptr).serialize(serializer), - ObType::Time => match Time::new(self.ptr, self.opts) { - Ok(val) => val.serialize(serializer), - Err(TimeError::HasTimezone) => err!(SerializeError::TimeHasTzinfo), - }, - ObType::Uuid => UUID::new(self.ptr).serialize(serializer), - ObType::Dict => { - if unlikely!(self.recursion == RECURSION_LIMIT) { - err!(SerializeError::RecursionLimit) - } - if unlikely!(unsafe { PyDict_GET_SIZE(self.ptr) } == 0) { - serializer.serialize_map(Some(0)).unwrap().end() - } else if self.opts & SORT_OR_NON_STR_KEYS == 0 { - Dict::new( - self.ptr, - self.opts, - self.default_calls, - self.recursion, - self.default, - ) - .serialize(serializer) - } else if self.opts & NON_STR_KEYS != 0 { - DictNonStrKey::new( - self.ptr, - self.opts, - self.default_calls, - self.recursion, - self.default, - ) - .serialize(serializer) - } else { - DictSortedKey::new( - self.ptr, - self.opts, - self.default_calls, - self.recursion, - self.default, - ) - .serialize(serializer) - } - } - ObType::List => { - if unlikely!(self.recursion == RECURSION_LIMIT) { - err!(SerializeError::RecursionLimit) + ObType::StrSubclass => { + StrSubclassSerializer::new(PyStrSubclassRef::from_ptr_unchecked(self.ptr)) + .serialize(serializer) } - if unlikely!(ffi!(PyList_GET_SIZE(self.ptr)) == 0) { - serializer.serialize_seq(Some(0)).unwrap().end() - } else { - ListSerializer::new( - self.ptr, - self.opts, - self.default_calls, - self.recursion, - self.default, - ) - .serialize(serializer) + ObType::Int => IntSerializer::new( + unsafe { PyIntRef::from_ptr_unchecked(self.ptr) }, + self.state.opts(), + ) + .serialize(serializer), + ObType::None => NoneSerializer::new().serialize(serializer), + ObType::Float => FloatSerializer::new(PyFloatRef::from_ptr_unchecked(self.ptr)) + .serialize(serializer), + ObType::Bool => BoolSerializer::new(PyBoolRef::from_ptr_unchecked(self.ptr)) + .serialize(serializer), + ObType::Datetime => DateTime::new( + PyDateTimeRef::from_ptr_unchecked(self.ptr), + self.state.opts(), + ) + .serialize(serializer), + ObType::Date => { + Date::new(PyDateRef::from_ptr_unchecked(self.ptr)).serialize(serializer) } - } - ObType::Tuple => TupleSerializer::new( - self.ptr, - self.opts, - self.default_calls, - self.recursion, - self.default, - ) - .serialize(serializer), - ObType::Dataclass => { - if unlikely!(self.recursion == RECURSION_LIMIT) { - err!(SerializeError::RecursionLimit) + ObType::Time => { + Time::new(PyTimeRef::from_ptr_unchecked(self.ptr), self.state.opts()) + .serialize(serializer) } - let dict = ffi!(PyObject_GetAttr(self.ptr, DICT_STR)); - let ob_type = ob_type!(self.ptr); - if unlikely!( - dict.is_null() || ffi!(PyDict_Contains((*ob_type).tp_dict, SLOTS_STR)) == 1 - ) { - unsafe { pyo3_ffi::PyErr_Clear() }; - DataclassFallbackSerializer::new( - self.ptr, - self.opts, - self.default_calls, - self.recursion, - self.default, - ) - .serialize(serializer) - } else { - ffi!(Py_DECREF(dict)); - DataclassFastSerializer::new( - dict, - self.opts, - self.default_calls, - self.recursion, - self.default, - ) - .serialize(serializer) + ObType::Uuid => { + UUID::new(PyUuidRef::from_ptr_unchecked(self.ptr)).serialize(serializer) } - } - ObType::Enum => { - let value = ffi!(PyObject_GetAttr(self.ptr, VALUE_STR)); - ffi!(Py_DECREF(value)); - PyObjectSerializer::new( - value, - self.opts, - self.default_calls, - self.recursion, + ObType::Dict => DictGenericSerializer::new( + PyDictRef::from_ptr_unchecked(self.ptr), + self.state, self.default, ) - .serialize(serializer) - } - ObType::NumpyArray => match NumpyArray::new(self.ptr, self.opts) { - Ok(val) => val.serialize(serializer), - Err(PyArrayError::Malformed) => err!(SerializeError::NumpyMalformed), - Err(PyArrayError::NotContiguous) | Err(PyArrayError::UnsupportedDataType) - if self.default.is_some() => - { - DefaultSerializer::new( - self.ptr, - self.opts, - self.default_calls, - self.recursion, - self.default, - ) - .serialize(serializer) + .serialize(serializer), + ObType::List => { + if ffi!(Py_SIZE(self.ptr)) == 0 { + ZeroListSerializer::new().serialize(serializer) + } else { + ListTupleSerializer::from_list( + PyListRef::from_ptr_unchecked(self.ptr), + self.state, + self.default, + ) + .serialize(serializer) + } + } + ObType::Tuple => { + if ffi!(Py_SIZE(self.ptr)) == 0 { + ZeroListSerializer::new().serialize(serializer) + } else { + ListTupleSerializer::from_tuple(self.ptr, self.state, self.default) + .serialize(serializer) + } } - Err(PyArrayError::NotContiguous) => { - err!(SerializeError::NumpyNotCContiguous) + ObType::Dataclass => DataclassGenericSerializer::new(self).serialize(serializer), + ObType::Enum => EnumSerializer::new(self).serialize(serializer), + ObType::NumpyArray => NumpySerializer::new(self).serialize(serializer), + ObType::NumpyScalar => { + NumpyScalar::new(self.ptr, self.state.opts()).serialize(serializer) } - Err(PyArrayError::UnsupportedDataType) => { - err!(SerializeError::NumpyUnsupportedDatatype) + ObType::Fragment => { + FragmentSerializer::new(unsafe { PyFragmentRef::from_ptr_unchecked(self.ptr) }) + .serialize(serializer) } - }, - ObType::NumpyScalar => NumpyScalar::new(self.ptr, self.opts).serialize(serializer), - ObType::Unknown => DefaultSerializer::new( - self.ptr, - self.opts, - self.default_calls, - self.recursion, - self.default, - ) - .serialize(serializer), + ObType::Unknown => DefaultSerializer::new(self).serialize(serializer), + } } } } diff --git a/src/serialize/state.rs b/src/serialize/state.rs new file mode 100644 index 00000000..eeada806 --- /dev/null +++ b/src/serialize/state.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2024-2025) + +use crate::opt::Opt; + +const RECURSION_SHIFT: usize = 24; +const RECURSION_MASK: u32 = 255 << RECURSION_SHIFT; + +const DEFAULT_SHIFT: usize = 16; +const DEFAULT_MASK: u32 = 255 << DEFAULT_SHIFT; + +#[repr(transparent)] +#[derive(Copy, Clone)] +pub(crate) struct SerializerState { + // recursion: u8, + // default_calls: u8, + // opts: u16, + state: u32, +} + +impl SerializerState { + #[inline(always)] + pub fn new(opts: Opt) -> Self { + debug_assert!(opts < u32::from(u16::MAX)); + Self { state: opts } + } + + #[inline(always)] + pub fn opts(self) -> u32 { + self.state + } + + #[inline(always)] + pub fn recursion_limit(self) -> bool { + self.state & RECURSION_MASK == RECURSION_MASK + } + + #[inline(always)] + pub fn default_calls_limit(self) -> bool { + self.state & DEFAULT_MASK == DEFAULT_MASK + } + + #[inline(always)] + pub fn copy_for_recursive_call(self) -> Self { + let opt = self.state & !RECURSION_MASK; + let recursion = (((self.state & RECURSION_MASK) >> RECURSION_SHIFT) + 1) << RECURSION_SHIFT; + Self { + state: opt | recursion, + } + } + + #[inline(always)] + pub fn copy_for_default_call(self) -> Self { + let opt = self.state & !DEFAULT_MASK; + let default_calls = (((self.state & DEFAULT_MASK) >> DEFAULT_SHIFT) + 1) << DEFAULT_SHIFT; + Self { + state: opt | default_calls, + } + } +} diff --git a/src/serialize/str.rs b/src/serialize/str.rs deleted file mode 100644 index ff0c9a6a..00000000 --- a/src/serialize/str.rs +++ /dev/null @@ -1,55 +0,0 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -use crate::serialize::error::*; -use crate::unicode::*; - -use serde::ser::{Serialize, Serializer}; - -#[repr(transparent)] -pub struct StrSerializer { - ptr: *mut pyo3_ffi::PyObject, -} - -impl StrSerializer { - pub fn new(ptr: *mut pyo3_ffi::PyObject) -> Self { - StrSerializer { ptr: ptr } - } -} - -impl Serialize for StrSerializer { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let uni = unicode_to_str(self.ptr); - if unlikely!(uni.is_none()) { - err!(SerializeError::InvalidStr) - } - serializer.serialize_str(uni.unwrap()) - } -} - -#[repr(transparent)] -pub struct StrSubclassSerializer { - ptr: *mut pyo3_ffi::PyObject, -} - -impl StrSubclassSerializer { - pub fn new(ptr: *mut pyo3_ffi::PyObject) -> Self { - StrSubclassSerializer { ptr: ptr } - } -} - -impl Serialize for StrSubclassSerializer { - #[inline(never)] - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let uni = unicode_to_str_via_ffi(self.ptr); - if unlikely!(uni.is_none()) { - err!(SerializeError::InvalidStr) - } - serializer.serialize_str(uni.unwrap()) - } -} diff --git a/src/serialize/tuple.rs b/src/serialize/tuple.rs deleted file mode 100644 index 662d04c7..00000000 --- a/src/serialize/tuple.rs +++ /dev/null @@ -1,54 +0,0 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -use crate::opt::*; -use crate::serialize::serializer::*; - -use serde::ser::{Serialize, SerializeSeq, Serializer}; -use std::ptr::NonNull; - -pub struct TupleSerializer { - ptr: *mut pyo3_ffi::PyObject, - opts: Opt, - default_calls: u8, - recursion: u8, - default: Option>, -} - -impl TupleSerializer { - pub fn new( - ptr: *mut pyo3_ffi::PyObject, - opts: Opt, - default_calls: u8, - recursion: u8, - default: Option>, - ) -> Self { - TupleSerializer { - ptr: ptr, - opts: opts, - default_calls: default_calls, - recursion: recursion, - default: default, - } - } -} - -impl Serialize for TupleSerializer { - #[inline(never)] - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut seq = serializer.serialize_seq(None).unwrap(); - for i in 0..=ffi!(PyTuple_GET_SIZE(self.ptr)) - 1 { - let elem = nonnull!(ffi!(PyTuple_GET_ITEM(self.ptr, i as isize))); - seq.serialize_element(&PyObjectSerializer::new( - elem.as_ptr(), - self.opts, - self.default_calls, - self.recursion + 1, - self.default, - ))? - } - seq.end() - } -} diff --git a/src/serialize/uuid.rs b/src/serialize/uuid.rs index 1c25ea14..7a8308f7 100644 --- a/src/serialize/uuid.rs +++ b/src/serialize/uuid.rs @@ -1,62 +1,20 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) -use crate::typeref::*; -use serde::ser::{Serialize, Serializer}; -use std::io::Write; -use std::os::raw::c_uchar; +use crate::ffi::PyUuidRef; +use crate::serialize::writer::{WriteExt, format_hyphenated}; -pub type UUIDBuffer = arrayvec::ArrayVec; - -pub struct UUID { - ptr: *mut pyo3_ffi::PyObject, -} - -impl UUID { - pub fn new(ptr: *mut pyo3_ffi::PyObject) -> Self { - UUID { ptr: ptr } - } - pub fn write_buf(&self, buf: &mut UUIDBuffer) { - let value: u128; - { - // test_uuid_immutable, test_uuid_int - let py_int = ffi!(PyObject_GetAttr(self.ptr, INT_ATTR_STR)); - ffi!(Py_DECREF(py_int)); - let buffer: [c_uchar; 16] = [0; 16]; - unsafe { - // test_uuid_overflow - pyo3_ffi::_PyLong_AsByteArray( - py_int as *mut pyo3_ffi::PyLongObject, - buffer.as_ptr() as *mut c_uchar, - 16, - 1, // little_endian - 0, // is_signed - ) - }; - value = u128::from_le_bytes(buffer); - } - - let mut hexadecimal = arrayvec::ArrayVec::::new(); - write!(hexadecimal, "{:032x}", value).unwrap(); - - buf.try_extend_from_slice(&hexadecimal[..8]).unwrap(); - buf.push(b'-'); - buf.try_extend_from_slice(&hexadecimal[8..12]).unwrap(); - buf.push(b'-'); - buf.try_extend_from_slice(&hexadecimal[12..16]).unwrap(); - buf.push(b'-'); - buf.try_extend_from_slice(&hexadecimal[16..20]).unwrap(); - buf.push(b'-'); - buf.try_extend_from_slice(&hexadecimal[20..32]).unwrap(); - } -} -impl Serialize for UUID { - #[inline(never)] - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut buf = arrayvec::ArrayVec::::new(); - self.write_buf(&mut buf); - serializer.serialize_str(str_from_slice!(buf.as_ptr(), buf.len())) +#[cold] +#[inline(never)] +pub(crate) fn write_uuid(ob: PyUuidRef, buf: &mut B) +where + B: ?Sized + WriteExt + bytes::BufMut, +{ + unsafe { + const UUID_LENGTH: usize = 36; + debug_assert!(buf.remaining_mut() >= UUID_LENGTH); + let dst: &mut [u8; UUID_LENGTH] = &mut *buf.as_mut_buffer_ptr().cast::<[u8; UUID_LENGTH]>(); + format_hyphenated(ob, dst); + buf.advance_mut(UUID_LENGTH); } } diff --git a/src/serialize/writer.rs b/src/serialize/writer.rs deleted file mode 100644 index 2a2e783d..00000000 --- a/src/serialize/writer.rs +++ /dev/null @@ -1,92 +0,0 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -use crate::ffi::PyBytesObject; -use core::ptr::NonNull; -use pyo3_ffi::*; -use std::os::raw::c_char; - -const BUFFER_LENGTH: usize = 1024; - -pub struct BytesWriter { - cap: usize, - len: usize, - bytes: *mut PyBytesObject, -} - -impl BytesWriter { - pub fn default() -> Self { - BytesWriter { - cap: BUFFER_LENGTH, - len: 0, - bytes: unsafe { - PyBytes_FromStringAndSize(std::ptr::null_mut(), BUFFER_LENGTH as isize) - as *mut PyBytesObject - }, - } - } - - pub fn finish(&mut self) -> NonNull { - unsafe { - unsafe { - std::ptr::write(self.buffer_ptr(), 0); - }; - (*self.bytes.cast::()).ob_size = self.len as Py_ssize_t; - self.resize(self.len as isize); - NonNull::new_unchecked(self.bytes as *mut PyObject) - } - } - - fn buffer_ptr(&self) -> *mut u8 { - unsafe { - std::mem::transmute::<*mut [c_char; 1], *mut u8>(std::ptr::addr_of_mut!( - (*self.bytes).ob_sval - )) - .add(self.len) - } - } - - pub fn resize(&mut self, len: isize) { - unsafe { - _PyBytes_Resize( - std::ptr::addr_of_mut!(self.bytes) as *mut *mut PyBytesObject as *mut *mut PyObject, - len as isize, - ); - } - } - - #[cold] - fn grow(&mut self, len: usize) { - while len >= self.cap { - if len < 262144 { - self.cap *= 4; - } else { - self.cap *= 2; - } - } - self.resize(self.cap as isize); - } -} - -impl std::io::Write for BytesWriter { - fn write(&mut self, buf: &[u8]) -> Result { - let _ = self.write_all(buf); - Ok(buf.len()) - } - - fn write_all(&mut self, buf: &[u8]) -> Result<(), std::io::Error> { - let to_write = buf.len(); - let end_length = self.len + to_write; - if unlikely!(end_length > self.cap) { - self.grow(end_length); - } - unsafe { - std::ptr::copy_nonoverlapping(buf.as_ptr() as *const u8, self.buffer_ptr(), to_write); - }; - self.len = end_length; - Ok(()) - } - - fn flush(&mut self) -> Result<(), std::io::Error> { - Ok(()) - } -} diff --git a/src/serialize/writer/byteswriter.rs b/src/serialize/writer/byteswriter.rs new file mode 100644 index 00000000..a548d9f6 --- /dev/null +++ b/src/serialize/writer/byteswriter.rs @@ -0,0 +1,266 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2020-2026) + +use crate::ffi::{PyBytes_FromStringAndSize, PyObject}; +use crate::util::usize_to_isize; +use bytes::{BufMut, buf::UninitSlice}; +use core::mem::MaybeUninit; +use core::ptr::NonNull; + +#[cfg(CPython)] +const BUFFER_LENGTH: usize = 1024; + +#[cfg(not(CPython))] +const BUFFER_LENGTH: usize = 4096; + +const OVERALLOCATION: usize = 64; + +pub(crate) struct BytesWriter { + cap: usize, + len: usize, + #[cfg(CPython)] + bytes: *mut crate::ffi::PyBytesObject, + #[cfg(not(CPython))] + bytes: *mut u8, +} + +impl BytesWriter { + #[inline] + pub fn default() -> Self { + BytesWriter { + cap: BUFFER_LENGTH, + len: 0, + #[cfg(CPython)] + bytes: unsafe { + PyBytes_FromStringAndSize(core::ptr::null_mut(), usize_to_isize(BUFFER_LENGTH)) + .cast::() + }, + #[cfg(not(CPython))] + bytes: unsafe { crate::ffi::PyMem_Malloc(BUFFER_LENGTH).cast::() }, + } + } + + #[cfg(CPython)] + pub fn abort(&mut self) { + unsafe { + crate::ffi::Py_DECREF(self.bytes.cast::()); + } + } + + #[cfg(not(CPython))] + pub fn abort(&mut self) { + unsafe { + crate::ffi::PyMem_Free(self.bytes.cast::()); + } + } + + fn append_and_terminate(&mut self, append: bool) { + unsafe { + if append { + core::ptr::write(self.buffer_ptr(), b'\n'); + self.len += 1; + } + #[cfg(CPython)] + core::ptr::write(self.buffer_ptr(), 0); + } + } + + #[cfg(CPython)] + #[inline] + pub fn finish(&mut self, append: bool) -> NonNull { + unsafe { + self.append_and_terminate(append); + crate::ffi::Py_SET_SIZE( + self.bytes.cast::(), + usize_to_isize(self.len), + ); + self.resize(self.len); + NonNull::new_unchecked(self.bytes.cast::()) + } + } + + #[cfg(not(CPython))] + #[inline] + pub fn finish(&mut self, append: bool) -> NonNull { + unsafe { + self.append_and_terminate(append); + let bytes = PyBytes_FromStringAndSize( + self.bytes.cast::().cast_const(), + usize_to_isize(self.len), + ); + debug_assert!(!bytes.is_null()); + crate::ffi::PyMem_Free(self.bytes.cast::()); + nonnull!(bytes) + } + } + + #[cfg(CPython)] + #[inline] + fn buffer_ptr(&self) -> *mut u8 { + unsafe { (&raw mut (*self.bytes).ob_sval).cast::().add(self.len) } + } + + #[cfg(not(CPython))] + #[inline] + fn buffer_ptr(&self) -> *mut u8 { + debug_assert!(!self.bytes.is_null()); + unsafe { self.bytes.add(self.len) } + } + + #[cfg(CPython)] + #[inline] + pub fn resize(&mut self, len: usize) { + self.cap = len; + unsafe { + crate::ffi::_PyBytes_Resize( + (&raw mut self.bytes).cast::<*mut PyObject>(), + usize_to_isize(len), + ); + } + } + + #[cfg(not(CPython))] + #[inline] + pub fn resize(&mut self, len: usize) { + self.cap = len; + unsafe { + self.bytes = + crate::ffi::PyMem_Realloc(self.bytes.cast::(), len).cast::(); + debug_assert!(!self.bytes.is_null()); + } + } + + #[cold] + #[inline(never)] + fn grow(&mut self, len: usize) { + let mut cap = self.cap; + while len >= cap { + cap *= 2; + } + self.resize(cap); + } +} + +unsafe impl BufMut for BytesWriter { + #[inline] + unsafe fn advance_mut(&mut self, cnt: usize) { + self.len += cnt; + } + + #[inline] + fn chunk_mut(&mut self) -> &mut UninitSlice { + unsafe { + UninitSlice::uninit(core::slice::from_raw_parts_mut( + self.buffer_ptr().cast::>(), + self.remaining_mut(), + )) + } + } + + #[inline] + fn remaining_mut(&self) -> usize { + self.cap - self.len + } + + #[inline] + fn put_u8(&mut self, value: u8) { + debug_assert!(self.remaining_mut() > 8); + unsafe { + core::ptr::write(self.buffer_ptr(), value); + self.advance_mut(1); + } + } + + #[inline] + fn put_bytes(&mut self, val: u8, cnt: usize) { + debug_assert!(self.remaining_mut() > cnt); + debug_assert!(self.remaining_mut() > 8); + unsafe { + core::ptr::write_bytes(self.buffer_ptr(), val, cnt); + self.advance_mut(cnt); + }; + } + + #[inline] + fn put_slice(&mut self, src: &[u8]) { + let len = src.len(); + debug_assert!(self.remaining_mut() > len); + debug_assert!(self.remaining_mut() > 8); + unsafe { + core::ptr::copy_nonoverlapping(src.as_ptr(), self.buffer_ptr(), len); + self.advance_mut(len); + } + } +} + +// hack based on saethlin's research and patch in https://github.com/serde-rs/json/issues/766 +pub(crate) trait WriteExt { + fn as_mut_buffer_ptr(&mut self) -> *mut u8; + + fn reserve(&mut self, len: usize); + + fn reserve_minimum(&mut self); + fn put_bool(&mut self, val: bool); + fn put_null(&mut self); +} + +impl WriteExt for &mut BytesWriter { + #[inline(always)] + fn as_mut_buffer_ptr(&mut self) -> *mut u8 { + self.buffer_ptr() + } + + #[inline(always)] + fn reserve(&mut self, len: usize) { + let end_length = self.len + len; + if end_length >= self.cap { + cold_path!(); + self.grow(end_length); + } + } + + #[inline] + fn reserve_minimum(&mut self) { + self.reserve(OVERALLOCATION * 2); + } + + #[cfg(feature = "inline_int")] + #[inline] + fn put_bool(&mut self, val: bool) { + debug_assert!(self.cap - self.len > 8); + unsafe { + const TRUE: (u64, usize) = (u64::from_ne_bytes(*b"true0000"), 4); + const FALSE: (u64, usize) = (u64::from_ne_bytes(*b"false000"), 5); + let (pattern, len) = core::hint::select_unpredictable(val, TRUE, FALSE); + #[allow(clippy::cast_ptr_alignment)] + core::ptr::write(self.buffer_ptr().cast::(), pattern); + self.advance_mut(len); + } + } + + #[cfg(not(feature = "inline_int"))] + #[inline] + fn put_bool(&mut self, val: bool) { + debug_assert!(self.cap - self.len > 8); + self.put_slice(core::hint::select_unpredictable(val, b"true", b"false")); + } + + #[cfg(feature = "inline_int")] + #[inline] + fn put_null(&mut self) { + debug_assert!(self.cap - self.len > 8); + unsafe { + const VAL: u32 = u32::from_ne_bytes(*b"null"); + #[allow(clippy::cast_ptr_alignment)] + core::ptr::write(self.buffer_ptr().cast::(), VAL); + self.advance_mut(4); + } + } + + #[cfg(not(feature = "inline_int"))] + #[inline] + fn put_null(&mut self) { + debug_assert!(self.cap - self.len > 8); + self.put_slice(b"null"); + } +} diff --git a/src/serialize/writer/format_str.rs b/src/serialize/writer/format_str.rs new file mode 100644 index 00000000..8c868419 --- /dev/null +++ b/src/serialize/writer/format_str.rs @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2024-2026) + +use crate::serialize::writer::byteswriter::WriteExt; + +macro_rules! reserve_str { + ($writer:expr, $value:expr) => { + $writer.reserve($value.len() * 8 + 32); + }; +} + +#[cfg(all( + target_arch = "x86_64", + feature = "avx512", + not(target_feature = "avx512vl") +))] +type StrFormatter = unsafe fn(*mut u8, *const u8, usize) -> usize; + +pub(crate) fn set_str_formatter_fn() { + unsafe { + #[cfg(all( + target_arch = "x86_64", + feature = "avx512", + not(target_feature = "avx512vl") + ))] + if std::is_x86_feature_detected!("avx512vl") { + STR_FORMATTER_FN = crate::serialize::writer::str::format_escaped_str_impl_512vl; + } + } +} + +#[cfg(all( + target_arch = "x86_64", + feature = "avx512", + not(target_feature = "avx512vl") +))] +static mut STR_FORMATTER_FN: StrFormatter = + crate::serialize::writer::str::format_escaped_str_impl_sse2_128; + +#[cfg(all( + target_arch = "x86_64", + feature = "avx512", + target_feature = "avx512vl" +))] +#[inline(always)] +pub(crate) fn format_escaped_str(writer: &mut W, value: &str) +where + W: ?Sized + WriteExt + bytes::BufMut, +{ + unsafe { + reserve_str!(writer, value); + + let written = crate::serialize::writer::str::format_escaped_str_impl_512vl( + writer.as_mut_buffer_ptr(), + value.as_bytes().as_ptr(), + value.len(), + ); + + writer.advance_mut(written); + } +} +#[cfg(all(target_arch = "x86_64", not(feature = "avx512")))] +#[inline(always)] +pub(crate) fn format_escaped_str(writer: &mut W, value: &str) +where + W: ?Sized + WriteExt + bytes::BufMut, +{ + unsafe { + reserve_str!(writer, value); + + let written = crate::serialize::writer::str::format_escaped_str_impl_sse2_128( + writer.as_mut_buffer_ptr(), + value.as_bytes().as_ptr(), + value.len(), + ); + + writer.advance_mut(written); + } +} + +#[cfg(all( + target_arch = "x86_64", + feature = "avx512", + not(target_feature = "avx512vl") +))] +#[inline(always)] +pub(crate) fn format_escaped_str(writer: &mut W, value: &str) +where + W: ?Sized + WriteExt + bytes::BufMut, +{ + unsafe { + reserve_str!(writer, value); + + let written = STR_FORMATTER_FN( + writer.as_mut_buffer_ptr(), + value.as_bytes().as_ptr(), + value.len(), + ); + + writer.advance_mut(written); + } +} + +#[cfg(all( + not(target_arch = "x86_64"), + not(feature = "avx512"), + feature = "generic_simd" +))] +#[inline(always)] +pub(crate) fn format_escaped_str(writer: &mut W, value: &str) +where + W: ?Sized + WriteExt + bytes::BufMut, +{ + unsafe { + reserve_str!(writer, value); + + let written = crate::serialize::writer::str::format_escaped_str_impl_generic_128( + writer.as_mut_buffer_ptr(), + value.as_bytes().as_ptr(), + value.len(), + ); + + writer.advance_mut(written); + } +} + +#[cfg(all(not(target_arch = "x86_64"), not(feature = "generic_simd")))] +#[inline(always)] +pub(crate) fn format_escaped_str(writer: &mut W, value: &str) +where + W: ?Sized + WriteExt + bytes::BufMut, +{ + unsafe { + reserve_str!(writer, value); + + let written = crate::serialize::writer::str::format_escaped_str_scalar( + writer.as_mut_buffer_ptr(), + value.as_bytes().as_ptr(), + value.len(), + ); + + writer.advance_mut(written); + } +} diff --git a/src/serialize/writer/formatter.rs b/src/serialize/writer/formatter.rs new file mode 100644 index 00000000..bc500272 --- /dev/null +++ b/src/serialize/writer/formatter.rs @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2022-2026) +// This is an adaptation of `src/value/ser.rs` from serde-json. + +use super::{ + write_float32, write_float64, write_integer_i32, write_integer_i64, write_integer_u32, + write_integer_u64, +}; +use crate::serialize::writer::WriteExt; +use std::io; + +macro_rules! debug_assert_has_capacity { + ($writer:expr) => { + debug_assert!($writer.remaining_mut() > 4) + }; +} + +pub(crate) trait Formatter { + #[inline] + fn write_null(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + writer.reserve_minimum(); + writer.put_null(); + Ok(()) + } + + #[inline] + fn write_bool(&mut self, writer: &mut W, value: bool) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + writer.reserve_minimum(); + writer.put_bool(value); + Ok(()) + } + + #[inline] + fn write_i32(&mut self, writer: &mut W, value: i32) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + writer.reserve_minimum(); + write_integer_i32(writer, value); + Ok(()) + } + + #[inline] + fn write_i64(&mut self, writer: &mut W, value: i64) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + writer.reserve_minimum(); + write_integer_i64(writer, value); + Ok(()) + } + + #[inline] + fn write_u32(&mut self, writer: &mut W, value: u32) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + writer.reserve_minimum(); + write_integer_u32(writer, value); + Ok(()) + } + + #[inline] + fn write_u64(&mut self, writer: &mut W, value: u64) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + writer.reserve_minimum(); + write_integer_u64(writer, value); + Ok(()) + } + + #[inline] + fn write_f32(&mut self, writer: &mut W, value: f32) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + writer.reserve_minimum(); + write_float32(writer, value); + Ok(()) + } + + #[inline] + fn write_f64(&mut self, writer: &mut W, value: f64) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + writer.reserve_minimum(); + write_float64(writer, value); + Ok(()) + } + + #[inline] + fn begin_array(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + writer.reserve_minimum(); + writer.put_u8(b'['); + Ok(()) + } + + #[inline] + fn end_array(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + debug_assert_has_capacity!(writer); + writer.put_u8(b']'); + Ok(()) + } + + #[inline] + fn begin_array_value(&mut self, writer: &mut W, first: bool) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + debug_assert_has_capacity!(writer); + if !first { + writer.put_u8(b','); + } + Ok(()) + } + + #[inline] + fn end_array_value(&mut self, _writer: &mut W) -> io::Result<()> + where + W: ?Sized, + { + Ok(()) + } + + #[inline] + fn begin_object(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + writer.reserve_minimum(); + writer.put_u8(b'{'); + Ok(()) + } + + #[inline] + fn end_object(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + writer.reserve_minimum(); + writer.put_u8(b'}'); + Ok(()) + } + + #[inline] + fn begin_object_key(&mut self, writer: &mut W, first: bool) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + debug_assert_has_capacity!(writer); + if !first { + writer.put_u8(b','); + } + Ok(()) + } + + #[inline] + fn end_object_key(&mut self, _writer: &mut W) -> io::Result<()> + where + W: ?Sized, + { + Ok(()) + } + + #[inline] + fn begin_object_value(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + debug_assert_has_capacity!(writer); + writer.put_u8(b':'); + Ok(()) + } + + #[inline] + fn end_object_value(&mut self, _writer: &mut W) -> io::Result<()> + where + W: ?Sized, + { + Ok(()) + } +} + +pub(crate) struct CompactFormatter; + +impl Formatter for CompactFormatter {} + +pub(crate) struct PrettyFormatter { + current_indent: usize, + has_value: bool, +} + +impl PrettyFormatter { + #[allow(clippy::new_without_default)] + pub const fn new() -> Self { + PrettyFormatter { + current_indent: 0, + has_value: false, + } + } +} + +impl Formatter for PrettyFormatter { + #[inline] + fn begin_array(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + self.current_indent += 1; + self.has_value = false; + writer.reserve_minimum(); + writer.put_u8(b'['); + Ok(()) + } + + #[inline] + fn end_array(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + self.current_indent -= 1; + let num_spaces = self.current_indent * 2; + writer.reserve(num_spaces + 32); + + if self.has_value { + writer.put_u8(b'\n'); + writer.put_bytes(b' ', num_spaces); + } + writer.put_u8(b']'); + Ok(()) + } + + #[inline] + fn begin_array_value(&mut self, writer: &mut W, first: bool) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + let num_spaces = self.current_indent * 2; + writer.reserve(num_spaces + 32); + + writer.put_slice(if first { b"\n" } else { b",\n" }); + writer.put_bytes(b' ', num_spaces); + Ok(()) + } + + #[inline] + fn end_array_value(&mut self, _writer: &mut W) -> io::Result<()> + where + W: ?Sized, + { + self.has_value = true; + Ok(()) + } + + #[inline] + fn begin_object(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + self.current_indent += 1; + self.has_value = false; + + writer.reserve_minimum(); + writer.put_u8(b'{'); + Ok(()) + } + + #[inline] + fn end_object(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + self.current_indent -= 1; + let num_spaces = self.current_indent * 2; + writer.reserve(num_spaces + 32); + + if self.has_value { + writer.put_u8(b'\n'); + writer.put_bytes(b' ', num_spaces); + } + + writer.put_u8(b'}'); + Ok(()) + } + + #[inline] + fn begin_object_key(&mut self, writer: &mut W, first: bool) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + let num_spaces = self.current_indent * 2; + writer.reserve(num_spaces + 32); + writer.put_slice(if first { b"\n" } else { b",\n" }); + writer.put_bytes(b' ', num_spaces); + Ok(()) + } + + #[inline] + fn begin_object_value(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + WriteExt + bytes::BufMut, + { + writer.reserve_minimum(); + writer.put_slice(b": "); + Ok(()) + } + + #[inline] + fn end_object_value(&mut self, _writer: &mut W) -> io::Result<()> + where + W: ?Sized, + { + self.has_value = true; + Ok(()) + } +} diff --git a/src/serialize/writer/half.rs b/src/serialize/writer/half.rs new file mode 100644 index 00000000..3ddf51c0 --- /dev/null +++ b/src/serialize/writer/half.rs @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright half-rs Contributors (2016-2026) +// https://github.com/VoidStarKat/half-rs + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[inline] +pub(crate) fn f16_to_f32(i: u16) -> f32 { + #[cfg(target_feature = "f16c")] + unsafe { + f16_to_f32_x86_f16c(i) + } + #[cfg(not(target_feature = "f16c"))] + unsafe { + if std::arch::is_x86_feature_detected!("f16c") { + f16_to_f32_x86_f16c(i) + } else { + cold_path!(); + f16_to_f32_fallback(i) + } + } +} + +#[cfg(target_arch = "aarch64")] +#[inline] +pub(crate) fn f16_to_f32(i: u16) -> f32 { + #[cfg(target_feature = "fp16")] + unsafe { + f16_to_f32_fp16(i) + } + #[cfg(not(target_feature = "fp16"))] + unsafe { + if std::arch::is_aarch64_feature_detected!("fp16") { + f16_to_f32_fp16(i) + } else { + cold_path!(); + f16_to_f32_fallback(i) + } + } +} + +#[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))] +#[inline] +pub(crate) fn f16_to_f32(i: u16) -> f32 { + f16_to_f32_fallback(i) +} + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[target_feature(enable = "f16c")] +#[inline] +unsafe fn f16_to_f32_x86_f16c(i: u16) -> f32 { + #[cfg(target_arch = "x86")] + use core::arch::x86::{__m128, __m128i, _mm_cvtph_ps}; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::{__m128, __m128i, _mm_cvtph_ps}; + use core::mem::transmute; + unsafe { + let vec: __m128i = transmute::<[u16; 8], __m128i>([i, 0, 0, 0, 0, 0, 0, 0]); + let retval: [f32; 4] = transmute::<__m128, [f32; 4]>(_mm_cvtph_ps(vec)); + retval[0] + } +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "fp16")] +#[inline] +unsafe fn f16_to_f32_fp16(i: u16) -> f32 { + unsafe { + let result: f32; + core::arch::asm!( + "fcvt {0:s}, {1:h}", + out(vreg) result, + in(vreg) i, + options(pure, nomem, nostack, preserves_flags)); + result + } +} + +#[inline] +#[allow(unused)] +const fn f16_to_f32_fallback(i: u16) -> f32 { + if i & 0x7FFFu16 == 0 { + return unsafe { f32::from_bits((i as u32) << 16) }; + } + let half_sign = (i & 0x8000u16) as u32; + let half_exp = (i & 0x7C00u16) as u32; + let half_man = (i & 0x03FFu16) as u32; + if half_exp == 0x7C00u32 { + if half_man == 0 { + return unsafe { f32::from_bits((half_sign << 16) | 0x7F80_0000u32) }; + } else { + return unsafe { + f32::from_bits((half_sign << 16) | 0x7FC0_0000u32 | (half_man << 13)) + }; + } + } + let sign = half_sign << 16; + let unbiased_exp = (half_exp.cast_signed() >> 10) - 15; + if half_exp == 0 { + let e = (half_man as u16).leading_zeros() - 6; + let exp = (127 - 15 - e) << 23; + let man = (half_man << (14 + e)) & 0x7F_FF_FFu32; + return unsafe { f32::from_bits(sign | exp | man) }; + } + let exp = (unbiased_exp + 127).cast_unsigned() << 23; + let man = (half_man & 0x03FFu32) << 13; + unsafe { f32::from_bits(sign | exp | man) } +} diff --git a/src/serialize/writer/json.rs b/src/serialize/writer/json.rs new file mode 100644 index 00000000..21235695 --- /dev/null +++ b/src/serialize/writer/json.rs @@ -0,0 +1,584 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2022-2026) +// This is an adaptation of `src/value/ser.rs` from serde-json. + +use super::format_str::format_escaped_str; +use crate::serialize::writer::WriteExt; +use crate::serialize::writer::formatter::{CompactFormatter, Formatter, PrettyFormatter}; +use serde::ser::{self, Impossible, Serialize}; +use serde_json::error::{Error, Result}; + +pub(crate) struct Serializer { + writer: W, + formatter: F, +} + +impl Serializer +where + W: WriteExt + bytes::BufMut, +{ + #[inline] + pub fn new(writer: W) -> Self { + Serializer::with_formatter(writer, CompactFormatter) + } +} + +impl Serializer +where + W: WriteExt + bytes::BufMut, +{ + #[inline] + pub fn pretty(writer: W) -> Self { + Serializer::with_formatter(writer, PrettyFormatter::new()) + } +} + +impl Serializer +where + W: WriteExt + bytes::BufMut, + F: Formatter, +{ + #[inline] + pub fn with_formatter(writer: W, formatter: F) -> Self { + Serializer { writer, formatter } + } +} + +impl<'a, W, F> ser::Serializer for &'a mut Serializer +where + W: WriteExt + bytes::BufMut, + F: Formatter, +{ + type Ok = (); + type Error = Error; + + type SerializeSeq = Compound<'a, W, F>; + type SerializeTuple = Impossible<(), Error>; + type SerializeTupleStruct = Impossible<(), Error>; + type SerializeTupleVariant = Impossible<(), Error>; + type SerializeMap = Compound<'a, W, F>; + type SerializeStruct = Impossible<(), Error>; + type SerializeStructVariant = Impossible<(), Error>; + + #[inline] + fn serialize_bool(self, value: bool) -> Result<()> { + self.formatter + .write_bool(&mut self.writer, value) + .map_err(Error::io) + } + + fn serialize_i8(self, _value: i8) -> Result<()> { + unreachable!(); + } + + fn serialize_i16(self, _value: i16) -> Result<()> { + unreachable!(); + } + + #[inline] + fn serialize_i32(self, value: i32) -> Result<()> { + self.formatter + .write_i32(&mut self.writer, value) + .map_err(Error::io) + } + + #[inline] + fn serialize_i64(self, value: i64) -> Result<()> { + self.formatter + .write_i64(&mut self.writer, value) + .map_err(Error::io) + } + + fn serialize_i128(self, _value: i128) -> Result<()> { + unreachable!(); + } + + fn serialize_u8(self, _value: u8) -> Result<()> { + unreachable!(); + } + + fn serialize_u16(self, _value: u16) -> Result<()> { + unreachable!(); + } + + #[inline] + fn serialize_u32(self, value: u32) -> Result<()> { + self.formatter + .write_u32(&mut self.writer, value) + .map_err(Error::io) + } + + #[inline] + fn serialize_u64(self, value: u64) -> Result<()> { + self.formatter + .write_u64(&mut self.writer, value) + .map_err(Error::io) + } + + fn serialize_u128(self, _value: u128) -> Result<()> { + unreachable!(); + } + + #[inline] + fn serialize_f32(self, value: f32) -> Result<()> { + if value.is_infinite() || value.is_nan() { + cold_path!(); + self.serialize_unit() + } else { + self.formatter + .write_f32(&mut self.writer, value) + .map_err(Error::io) + } + } + #[inline] + fn serialize_f64(self, value: f64) -> Result<()> { + if value.is_infinite() || value.is_nan() { + cold_path!(); + self.serialize_unit() + } else { + self.formatter + .write_f64(&mut self.writer, value) + .map_err(Error::io) + } + } + + fn serialize_char(self, _value: char) -> Result<()> { + unreachable!(); + } + + #[inline(always)] + fn serialize_str(self, value: &str) -> Result<()> { + format_escaped_str(&mut self.writer, value); + Ok(()) + } + + #[inline(always)] + fn serialize_bytes(self, value: &[u8]) -> Result<()> { + self.writer.reserve(value.len() + 32); + unsafe { + self.writer.put_slice(value); + } + Ok(()) + } + + #[inline] + fn serialize_unit(self) -> Result<()> { + self.formatter + .write_null(&mut self.writer) + .map_err(Error::io) + } + + #[inline(always)] + fn serialize_unit_struct(self, name: &'static str) -> Result<()> { + debug_assert!(name.len() <= 36); + self.writer.reserve_minimum(); + unsafe { + self.writer.put_u8(b'"'); + self.writer.put_slice(name.as_bytes()); + self.writer.put_u8(b'"'); + } + Ok(()) + } + + fn serialize_unit_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + ) -> Result<()> { + unreachable!(); + } + + fn serialize_newtype_struct(self, _name: &'static str, _value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + unreachable!(); + } + + fn serialize_newtype_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _value: &T, + ) -> Result<()> + where + T: ?Sized + Serialize, + { + unreachable!(); + } + + #[inline] + fn serialize_none(self) -> Result<()> { + self.serialize_unit() + } + + #[inline] + fn serialize_some(self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(self) + } + + #[inline(always)] + fn serialize_seq(self, _len: Option) -> Result { + self.formatter + .begin_array(&mut self.writer) + .map_err(Error::io)?; + Ok(Compound { + ser: self, + state: State::First, + }) + } + + fn serialize_tuple(self, _len: usize) -> Result { + unreachable!(); + } + + fn serialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + ) -> Result { + unreachable!(); + } + + fn serialize_tuple_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _len: usize, + ) -> Result { + unreachable!(); + } + + #[inline(always)] + fn serialize_map(self, _len: Option) -> Result { + self.formatter + .begin_object(&mut self.writer) + .map_err(Error::io)?; + Ok(Compound { + ser: self, + state: State::First, + }) + } + + fn serialize_struct(self, _name: &'static str, _len: usize) -> Result { + unreachable!(); + } + + fn serialize_struct_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _len: usize, + ) -> Result { + unreachable!(); + } +} + +#[derive(Eq, PartialEq)] +pub(crate) enum State { + First, + Rest, +} + +pub(crate) struct Compound<'a, W: 'a, F: 'a> { + ser: &'a mut Serializer, + state: State, +} + +impl ser::SerializeSeq for Compound<'_, W, F> +where + W: WriteExt + bytes::BufMut, + F: Formatter, +{ + type Ok = (); + type Error = Error; + + #[inline] + fn serialize_element(&mut self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + self.ser + .formatter + .begin_array_value(&mut self.ser.writer, self.state == State::First) + .unwrap(); + self.state = State::Rest; + value.serialize(&mut *self.ser)?; + self.ser + .formatter + .end_array_value(&mut self.ser.writer) + .map_err(Error::io) + .unwrap(); + Ok(()) + } + + #[inline] + fn end(self) -> Result<()> { + self.ser.formatter.end_array(&mut self.ser.writer).unwrap(); + Ok(()) + } +} + +impl ser::SerializeMap for Compound<'_, W, F> +where + W: WriteExt + bytes::BufMut, + F: Formatter, +{ + type Ok = (); + type Error = Error; + + fn serialize_entry(&mut self, _key: &K, _value: &V) -> Result<()> + where + K: ?Sized + Serialize, + V: ?Sized + Serialize, + { + unreachable!() + } + + #[inline] + fn serialize_key(&mut self, key: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + self.ser + .formatter + .begin_object_key(&mut self.ser.writer, self.state == State::First) + .unwrap(); + self.state = State::Rest; + + key.serialize(MapKeySerializer { ser: self.ser })?; + + self.ser + .formatter + .end_object_key(&mut self.ser.writer) + .unwrap(); + Ok(()) + } + + #[inline] + fn serialize_value(&mut self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + self.ser + .formatter + .begin_object_value(&mut self.ser.writer) + .unwrap(); + value.serialize(&mut *self.ser)?; + self.ser + .formatter + .end_object_value(&mut self.ser.writer) + .unwrap(); + Ok(()) + } + + #[inline] + fn end(self) -> Result<()> { + self.ser.formatter.end_object(&mut self.ser.writer).unwrap(); + Ok(()) + } +} + +#[repr(transparent)] +struct MapKeySerializer<'a, W: 'a, F: 'a> { + ser: &'a mut Serializer, +} + +impl ser::Serializer for MapKeySerializer<'_, W, F> +where + W: WriteExt + bytes::BufMut, + F: Formatter, +{ + type Ok = (); + type Error = Error; + type SerializeSeq = Impossible<(), Error>; + type SerializeTuple = Impossible<(), Error>; + type SerializeTupleStruct = Impossible<(), Error>; + type SerializeTupleVariant = Impossible<(), Error>; + type SerializeMap = Impossible<(), Error>; + type SerializeStruct = Impossible<(), Error>; + type SerializeStructVariant = Impossible<(), Error>; + + #[inline(always)] + fn serialize_str(self, value: &str) -> Result<()> { + self.ser.serialize_str(value) + } + + fn serialize_unit_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + ) -> Result<()> { + unreachable!(); + } + + fn serialize_newtype_struct(self, _name: &'static str, _value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + unreachable!(); + } + fn serialize_bool(self, _value: bool) -> Result<()> { + unreachable!(); + } + + fn serialize_i8(self, _value: i8) -> Result<()> { + unreachable!(); + } + + fn serialize_i16(self, _value: i16) -> Result<()> { + unreachable!(); + } + + fn serialize_i32(self, _value: i32) -> Result<()> { + unreachable!(); + } + + fn serialize_i64(self, _value: i64) -> Result<()> { + unreachable!(); + } + + fn serialize_i128(self, _value: i128) -> Result<()> { + unreachable!(); + } + + fn serialize_u8(self, _value: u8) -> Result<()> { + unreachable!(); + } + + fn serialize_u16(self, _value: u16) -> Result<()> { + unreachable!(); + } + + fn serialize_u32(self, _value: u32) -> Result<()> { + unreachable!(); + } + + fn serialize_u64(self, _value: u64) -> Result<()> { + unreachable!(); + } + + fn serialize_u128(self, _value: u128) -> Result<()> { + unreachable!(); + } + + fn serialize_f32(self, _value: f32) -> Result<()> { + unreachable!(); + } + + fn serialize_f64(self, _value: f64) -> Result<()> { + unreachable!(); + } + + fn serialize_char(self, _value: char) -> Result<()> { + unreachable!(); + } + + fn serialize_bytes(self, _value: &[u8]) -> Result<()> { + unreachable!(); + } + + fn serialize_unit(self) -> Result<()> { + unreachable!(); + } + + fn serialize_unit_struct(self, _name: &'static str) -> Result<()> { + unreachable!(); + } + + fn serialize_newtype_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _value: &T, + ) -> Result<()> + where + T: ?Sized + Serialize, + { + unreachable!(); + } + + fn serialize_none(self) -> Result<()> { + unreachable!(); + } + + fn serialize_some(self, _value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + unreachable!(); + } + + fn serialize_seq(self, _len: Option) -> Result { + unreachable!(); + } + + fn serialize_tuple(self, _len: usize) -> Result { + unreachable!(); + } + + fn serialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + ) -> Result { + unreachable!(); + } + + fn serialize_tuple_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _len: usize, + ) -> Result { + unreachable!(); + } + + fn serialize_map(self, _len: Option) -> Result { + unreachable!(); + } + + fn serialize_struct(self, _name: &'static str, _len: usize) -> Result { + unreachable!(); + } + + fn serialize_struct_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _len: usize, + ) -> Result { + unreachable!(); + } +} + +#[inline] +pub(crate) fn to_writer(writer: W, value: &T) -> Result<()> +where + W: WriteExt + bytes::BufMut, + T: ?Sized + Serialize, +{ + let mut ser = Serializer::new(writer); + value.serialize(&mut ser) +} + +#[inline] +pub(crate) fn to_writer_pretty(writer: W, value: &T) -> Result<()> +where + W: WriteExt + bytes::BufMut, + T: ?Sized + Serialize, +{ + let mut ser = Serializer::pretty(writer); + value.serialize(&mut ser) +} diff --git a/src/serialize/writer/mod.rs b/src/serialize/writer/mod.rs new file mode 100644 index 00000000..cfa74558 --- /dev/null +++ b/src/serialize/writer/mod.rs @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2024-2026) + +mod byteswriter; +mod format_str; +mod formatter; +mod half; +mod json; +mod num; +mod smallfixedbuffer; +mod str; +mod uuid; + +pub(crate) use byteswriter::{BytesWriter, WriteExt}; +pub(crate) use format_str::set_str_formatter_fn; +pub(crate) use half::f16_to_f32; +pub(crate) use json::{to_writer, to_writer_pretty}; +pub(crate) use num::{ + write_float32, write_float64, write_integer_i32, write_integer_i64, write_integer_u32, + write_integer_u64, +}; +pub(crate) use smallfixedbuffer::SmallFixedBuffer; +pub(crate) use uuid::format_hyphenated; diff --git a/src/serialize/writer/num.rs b/src/serialize/writer/num.rs new file mode 100644 index 00000000..9fb54c35 --- /dev/null +++ b/src/serialize/writer/num.rs @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2026) + +use crate::serialize::writer::WriteExt; +use bytes::BufMut; + +#[inline] +pub(crate) fn write_integer_u32(buf: &mut B, val: u32) +where + B: ?Sized + WriteExt + BufMut, +{ + write_integer(buf, val) +} + +#[inline] +pub(crate) fn write_integer_i32(buf: &mut B, val: i32) +where + B: ?Sized + WriteExt + BufMut, +{ + write_integer(buf, val) +} + +#[inline] +pub(crate) fn write_integer_u64(buf: &mut B, val: u64) +where + B: ?Sized + WriteExt + BufMut, +{ + write_integer(buf, val) +} + +#[inline] +pub(crate) fn write_integer_i64(buf: &mut B, val: i64) +where + B: ?Sized + WriteExt + BufMut, +{ + write_integer(buf, val) +} + +#[inline] +fn write_integer(buf: &mut B, val: V) +where + B: ?Sized + WriteExt + BufMut, +{ + unsafe { + debug_assert!(buf.remaining_mut() >= 20); + let len = itoap::write_to_ptr(buf.as_mut_buffer_ptr(), val); + buf.advance_mut(len); + } +} + +#[inline] +pub(crate) fn write_float32(buf: &mut B, val: f32) +where + B: ?Sized + WriteExt + BufMut, +{ + if val.is_infinite() || val.is_nan() { + cold_path!(); + buf.put_null(); + } else { + write_finite_float(buf, val) + } +} + +#[inline] +pub(crate) fn write_float64(buf: &mut B, val: f64) +where + B: ?Sized + WriteExt + BufMut, +{ + if val.is_infinite() || val.is_nan() { + cold_path!(); + buf.put_null(); + } else { + write_finite_float(buf, val) + } +} + +fn write_finite_float(buf: &mut B, val: F) +where + B: ?Sized + WriteExt + BufMut, +{ + unsafe { + debug_assert!(buf.remaining_mut() >= 40); + let buffer = &mut *buf.as_mut_buffer_ptr().cast::(); + let res = buffer.format_finite(val); + buf.advance_mut(res.len()); + } +} diff --git a/src/serialize/writer/smallfixedbuffer.rs b/src/serialize/writer/smallfixedbuffer.rs new file mode 100644 index 00000000..b3da892e --- /dev/null +++ b/src/serialize/writer/smallfixedbuffer.rs @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2023-2026) + +use crate::serialize::writer::WriteExt; +use bytes::{BufMut, buf::UninitSlice}; +use core::mem::MaybeUninit; + +const BUFFER_LENGTH: usize = 64 - core::mem::size_of::(); + +/// For use to serialize fixed-size UUIDs and DateTime. +#[repr(align(64))] +pub(crate) struct SmallFixedBuffer { + idx: usize, + bytes: [MaybeUninit; BUFFER_LENGTH], +} + +impl SmallFixedBuffer { + #[inline] + pub fn new() -> Self { + Self { + idx: 0, + bytes: [MaybeUninit::::uninit(); BUFFER_LENGTH], + } + } + + #[inline] + pub fn as_ptr(&self) -> *const u8 { + (&raw const self.bytes).cast::() + } + + #[inline] + pub fn len(&self) -> usize { + self.idx + } + + #[allow(clippy::inherent_to_string)] + #[inline] + pub fn to_string(&self) -> String { + String::from(str_from_slice!(self.as_ptr(), self.len())) + } +} + +unsafe impl BufMut for SmallFixedBuffer { + #[inline] + unsafe fn advance_mut(&mut self, cnt: usize) { + self.idx += cnt; + } + + #[inline] + fn chunk_mut(&mut self) -> &mut UninitSlice { + UninitSlice::uninit(&mut self.bytes) + } + + #[inline] + fn remaining_mut(&self) -> usize { + BUFFER_LENGTH - self.idx + } + + #[inline] + fn put_u8(&mut self, value: u8) { + debug_assert!(self.remaining_mut() > 8); + unsafe { + core::ptr::write((&raw mut self.bytes).cast::().add(self.idx), value); + self.advance_mut(1); + }; + } + + #[inline] + fn put_slice(&mut self, src: &[u8]) { + debug_assert!(self.remaining_mut() > src.len()); + unsafe { + core::ptr::copy_nonoverlapping( + src.as_ptr(), + (&raw mut self.bytes).cast::().add(self.idx), + src.len(), + ); + self.advance_mut(src.len()); + } + } +} + +impl WriteExt for SmallFixedBuffer { + #[inline(always)] + fn as_mut_buffer_ptr(&mut self) -> *mut u8 { + unsafe { self.as_ptr().cast_mut().add(self.idx) } + } + + fn reserve(&mut self, _len: usize) { + unimplemented!() + } + + fn reserve_minimum(&mut self) { + unimplemented!() + } + + fn put_bool(&mut self, _val: bool) { + unimplemented!() + } + + fn put_null(&mut self) { + unimplemented!() + } +} diff --git a/src/serialize/writer/str/avx512.rs b/src/serialize/writer/str/avx512.rs new file mode 100644 index 00000000..75001711 --- /dev/null +++ b/src/serialize/writer/str/avx512.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2024-2025) + +use core::arch::x86_64::{ + _mm256_cmpeq_epu8_mask, _mm256_cmplt_epu8_mask, _mm256_loadu_epi8, _mm256_maskz_loadu_epi8, + _mm256_set1_epi8, _mm256_storeu_epi8, +}; + +#[target_feature(enable = "avx512f,avx512bw,avx512vl,bmi2")] +pub(crate) unsafe fn format_escaped_str_impl_512vl( + odst: *mut u8, + value_ptr: *const u8, + value_len: usize, +) -> usize { + unsafe { + const STRIDE: usize = 32; + + let mut dst = odst; + let mut src = value_ptr; + let mut nb: usize = value_len; + + let blash = _mm256_set1_epi8(0b01011100i8); + let quote = _mm256_set1_epi8(0b00100010i8); + let x20 = _mm256_set1_epi8(0b00100000i8); + + core::ptr::write(dst, b'"'); + dst = dst.add(1); + + while nb >= STRIDE { + let str_vec = _mm256_loadu_epi8(src.cast::()); + + _mm256_storeu_epi8(dst.cast::(), str_vec); + + let mask = _mm256_cmpeq_epu8_mask(str_vec, blash) + | _mm256_cmpeq_epu8_mask(str_vec, quote) + | _mm256_cmplt_epu8_mask(str_vec, x20); + + if mask != 0 { + let cn = mask.trailing_zeros() as usize; + src = src.add(cn); + dst = dst.add(cn); + nb -= cn; + nb -= 1; + + write_escape!(*(src), dst); + src = src.add(1); + } else { + nb -= STRIDE; + dst = dst.add(STRIDE); + src = src.add(STRIDE); + } + } + + loop { + let remainder_mask = !(u32::MAX << nb); + let str_vec = _mm256_maskz_loadu_epi8(remainder_mask, src.cast::()); + + _mm256_storeu_epi8(dst.cast::(), str_vec); + + let mask = (_mm256_cmpeq_epu8_mask(str_vec, blash) + | _mm256_cmpeq_epu8_mask(str_vec, quote) + | _mm256_cmplt_epu8_mask(str_vec, x20)) + & remainder_mask; + + if mask != 0 { + let cn = mask.trailing_zeros() as usize; + src = src.add(cn); + dst = dst.add(cn); + nb -= cn; + nb -= 1; + + write_escape!(*(src), dst); + src = src.add(1); + } else { + dst = dst.add(nb); + break; + } + } + + core::ptr::write(dst, b'"'); + dst = dst.add(1); + + dst as usize - odst as usize + } +} diff --git a/src/serialize/writer/str/escape.rs b/src/serialize/writer/str/escape.rs new file mode 100644 index 00000000..f613d5c6 --- /dev/null +++ b/src/serialize/writer/str/escape.rs @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2024-2026) +// the constants and SIMD approach are adapted from cloudwego's sonic-rs + +#[cfg(feature = "inline_int")] +macro_rules! write_escape { + ($byte:expr, $dst:expr) => { + debug_assert!($byte < 96); + #[allow(unnecessary_transmutes)] + let escape = core::mem::transmute::<[u8; 8], u64>( + *crate::serialize::writer::str::escape::QUOTE_TAB.get_unchecked($byte as usize), + ); + #[allow(clippy::cast_ptr_alignment)] + let _dst = $dst.cast::(); // stmt_expr_attributes + core::ptr::write(_dst, escape); + $dst = $dst.add((escape as usize) >> 56); + }; +} + +#[cfg(not(feature = "inline_int"))] +macro_rules! write_escape { + ($byte:expr, $dst:expr) => { + debug_assert!($byte < 96); + let escape = crate::serialize::writer::str::escape::QUOTE_TAB.get_unchecked($byte as usize); + core::ptr::copy_nonoverlapping(escape.as_ptr(), $dst, 8); + $dst = $dst.add(((*escape.as_ptr().add(7)) as usize)); + }; +} + +pub(crate) const NEED_ESCAPED: [u8; 256] = [ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +]; + +pub(crate) const QUOTE_TAB: [[u8; 8]; 96] = [ + [b'\\', b'u', b'0', b'0', b'0', b'0', 0, 6], + [b'\\', b'u', b'0', b'0', b'0', b'1', 0, 6], + [b'\\', b'u', b'0', b'0', b'0', b'2', 0, 6], + [b'\\', b'u', b'0', b'0', b'0', b'3', 0, 6], + [b'\\', b'u', b'0', b'0', b'0', b'4', 0, 6], + [b'\\', b'u', b'0', b'0', b'0', b'5', 0, 6], + [b'\\', b'u', b'0', b'0', b'0', b'6', 0, 6], + [b'\\', b'u', b'0', b'0', b'0', b'7', 0, 6], + [b'\\', b'b', b'0', b'0', b'0', b'0', 0, 2], + [b'\\', b't', b'0', b'0', b'0', b'0', 0, 2], + [b'\\', b'n', b'0', b'0', b'0', b'0', 0, 2], + [b'\\', b'u', b'0', b'0', b'0', b'b', 0, 6], + [b'\\', b'f', b'0', b'0', b'0', b'0', 0, 2], + [b'\\', b'r', b'0', b'0', b'0', b'0', 0, 2], + [b'\\', b'u', b'0', b'0', b'0', b'e', 0, 6], + [b'\\', b'u', b'0', b'0', b'0', b'f', 0, 6], + [b'\\', b'u', b'0', b'0', b'1', b'0', 0, 6], + [b'\\', b'u', b'0', b'0', b'1', b'1', 0, 6], + [b'\\', b'u', b'0', b'0', b'1', b'2', 0, 6], + [b'\\', b'u', b'0', b'0', b'1', b'3', 0, 6], + [b'\\', b'u', b'0', b'0', b'1', b'4', 0, 6], + [b'\\', b'u', b'0', b'0', b'1', b'5', 0, 6], + [b'\\', b'u', b'0', b'0', b'1', b'6', 0, 6], + [b'\\', b'u', b'0', b'0', b'1', b'7', 0, 6], + [b'\\', b'u', b'0', b'0', b'1', b'8', 0, 6], + [b'\\', b'u', b'0', b'0', b'1', b'9', 0, 6], + [b'\\', b'u', b'0', b'0', b'1', b'a', 0, 6], + [b'\\', b'u', b'0', b'0', b'1', b'b', 0, 6], + [b'\\', b'u', b'0', b'0', b'1', b'c', 0, 6], + [b'\\', b'u', b'0', b'0', b'1', b'd', 0, 6], + [b'\\', b'u', b'0', b'0', b'1', b'e', 0, 6], + [b'\\', b'u', b'0', b'0', b'1', b'f', 0, 6], + [0; 8], + [0; 8], + [b'\\', b'"', 0, 0, 0, 0, 0, 2], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [0; 8], + [b'\\', b'\\', 0, 0, 0, 0, 0, 2], + [0; 8], + [0; 8], + [0; 8], +]; diff --git a/src/serialize/writer/str/generic.rs b/src/serialize/writer/str/generic.rs new file mode 100644 index 00000000..091e237a --- /dev/null +++ b/src/serialize/writer/str/generic.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2024-2025) + +use core::simd::cmp::{SimdPartialEq, SimdPartialOrd}; +use core::simd::u8x16; + +#[cfg_attr(target_arch = "aarch64", target_feature(enable = "neon"))] +pub(crate) unsafe fn format_escaped_str_impl_generic_128( + odst: *mut u8, + value_ptr: *const u8, + value_len: usize, +) -> usize { + unsafe { + const STRIDE: usize = 16; + + let mut dst = odst; + let mut src = value_ptr; + + core::ptr::write(dst, b'"'); + dst = dst.add(1); + + if value_len < STRIDE { + impl_format_scalar!(dst, src, value_len); + } else { + let blash = u8x16::splat(b'\\'); + let quote = u8x16::splat(b'"'); + let x20 = u8x16::splat(32); + + let last_stride_src = src.add(value_len).sub(STRIDE); + let mut nb: usize = value_len; + + { + while nb >= STRIDE { + let v = u8x16::from_slice(core::slice::from_raw_parts(src, STRIDE)); + let mask = + (v.simd_eq(blash) | v.simd_eq(quote) | v.simd_lt(x20)).to_bitmask() as u32; + v.copy_to_slice(core::slice::from_raw_parts_mut(dst, STRIDE)); + + if mask != 0 { + let cn = mask.trailing_zeros() as usize; + nb -= cn; + dst = dst.add(cn); + src = src.add(cn); + nb -= 1; + write_escape!(*(src), dst); + src = src.add(1); + } else { + nb -= STRIDE; + dst = dst.add(STRIDE); + src = src.add(STRIDE); + } + } + } + + let mut scratch: [u8; 32] = [b'a'; 32]; + let mut v = u8x16::from_slice(core::slice::from_raw_parts(last_stride_src, STRIDE)); + v.copy_to_slice(core::slice::from_raw_parts_mut( + scratch.as_mut_ptr(), + STRIDE, + )); + + let mut scratch_ptr = scratch.as_mut_ptr().add(16 - nb); + v = u8x16::from_slice(core::slice::from_raw_parts(scratch_ptr, STRIDE)); + let mut mask = + (v.simd_eq(blash) | v.simd_eq(quote) | v.simd_lt(x20)).to_bitmask() as u32; + + loop { + v.copy_to_slice(core::slice::from_raw_parts_mut(dst, STRIDE)); + if mask != 0 { + let cn = mask.trailing_zeros() as usize; + nb -= cn; + dst = dst.add(cn); + scratch_ptr = scratch_ptr.add(cn); + nb -= 1; + mask >>= cn + 1; + write_escape!(*(scratch_ptr), dst); + scratch_ptr = scratch_ptr.add(1); + v = u8x16::from_slice(core::slice::from_raw_parts(scratch_ptr, STRIDE)); + } else { + dst = dst.add(nb); + break; + } + } + } + + core::ptr::write(dst, b'"'); + dst = dst.add(1); + + dst as usize - odst as usize + } +} diff --git a/src/serialize/writer/str/mod.rs b/src/serialize/writer/str/mod.rs new file mode 100644 index 00000000..b560aaf1 --- /dev/null +++ b/src/serialize/writer/str/mod.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2024-2025) + +#[macro_use] +mod escape; +#[macro_use] +mod scalar; + +#[cfg(all(feature = "generic_simd", not(target_arch = "x86_64")))] +mod generic; + +#[cfg(target_arch = "x86_64")] +mod sse2; + +#[cfg(all(target_arch = "x86_64", feature = "avx512"))] +mod avx512; + +#[cfg(all(not(target_arch = "x86_64"), not(feature = "generic_simd")))] +pub(crate) use scalar::format_escaped_str_scalar; + +#[cfg(all(target_arch = "x86_64", feature = "avx512"))] +pub(crate) use avx512::format_escaped_str_impl_512vl; + +#[allow(unused_imports)] +#[cfg(target_arch = "x86_64")] +pub(crate) use sse2::format_escaped_str_impl_sse2_128; + +#[cfg(all(feature = "generic_simd", not(target_arch = "x86_64")))] +pub(crate) use generic::format_escaped_str_impl_generic_128; diff --git a/src/serialize/writer/str/scalar.rs b/src/serialize/writer/str/scalar.rs new file mode 100644 index 00000000..aed9fc0e --- /dev/null +++ b/src/serialize/writer/str/scalar.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2024-2026) + +macro_rules! impl_format_scalar { + ($dst:expr, $src:expr, $value_len:expr) => { + for _ in 0..$value_len { + core::ptr::write($dst, *($src)); + $src = $src.add(1); + $dst = $dst.add(1); + if *super::escape::NEED_ESCAPED.get_unchecked(*($src.sub(1)) as usize) != 0 { + $dst = $dst.sub(1); + write_escape!(*($src.sub(1)), $dst); + } + } + }; +} + +#[cfg(all(not(target_arch = "x86_64"), not(feature = "generic_simd")))] +pub(crate) unsafe fn format_escaped_str_scalar( + odst: *mut u8, + value_ptr: *const u8, + value_len: usize, +) -> usize { + unsafe { + let mut dst = odst; + let mut src = value_ptr; + + core::ptr::write(dst, b'"'); + dst = dst.add(1); + + impl_format_scalar!(dst, src, value_len); + + core::ptr::write(dst, b'"'); + dst = dst.add(1); + + dst as usize - odst as usize + } +} diff --git a/src/serialize/writer/str/sse2.rs b/src/serialize/writer/str/sse2.rs new file mode 100644 index 00000000..ca041005 --- /dev/null +++ b/src/serialize/writer/str/sse2.rs @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright ijl (2024-2026) + +use core::arch::x86_64::{ + __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, _mm_set1_epi8, + _mm_setzero_si128, _mm_storeu_si128, _mm_subs_epu8, +}; + +#[allow(dead_code)] +#[expect(clippy::cast_ptr_alignment)] +pub(crate) unsafe fn format_escaped_str_impl_sse2_128( + odst: *mut u8, + value_ptr: *const u8, + value_len: usize, +) -> usize { + unsafe { + const STRIDE: usize = 16; + + let mut dst = odst; + let mut src = value_ptr; + + core::ptr::write(dst, b'"'); + dst = dst.add(1); + + if value_len < STRIDE { + impl_format_scalar!(dst, src, value_len); + } else { + let blash = _mm_set1_epi8(0b01011100i8); + let quote = _mm_set1_epi8(0b00100010i8); + let x20 = _mm_set1_epi8(0b00011111i8); + let v0 = _mm_setzero_si128(); + + let last_stride_src = src.add(value_len).sub(STRIDE); + let mut nb: usize = value_len; + + while nb >= STRIDE { + let str_vec = _mm_loadu_si128(src.cast::<__m128i>()); + + let mask = _mm_movemask_epi8(_mm_or_si128( + _mm_or_si128( + _mm_cmpeq_epi8(str_vec, blash), + _mm_cmpeq_epi8(str_vec, quote), + ), + _mm_cmpeq_epi8(_mm_subs_epu8(str_vec, x20), v0), + )); + + _mm_storeu_si128(dst.cast::<__m128i>(), str_vec); + + if mask != 0 { + let cn = mask.trailing_zeros() as usize; + nb -= cn; + dst = dst.add(cn); + src = src.add(cn); + nb -= 1; + write_escape!(*(src), dst); + src = src.add(1); + } else { + nb -= STRIDE; + dst = dst.add(STRIDE); + src = src.add(STRIDE); + } + } + + let mut scratch: [u8; 32] = [b'a'; 32]; + let mut str_vec = _mm_loadu_si128(last_stride_src.cast::<__m128i>()); + _mm_storeu_si128(scratch.as_mut_ptr().cast::<__m128i>(), str_vec); + + let mut scratch_ptr = scratch.as_mut_ptr().add(16 - nb); + str_vec = _mm_loadu_si128(scratch_ptr as *const __m128i); + + let mut mask = _mm_movemask_epi8(_mm_or_si128( + _mm_or_si128( + _mm_cmpeq_epi8(str_vec, blash), + _mm_cmpeq_epi8(str_vec, quote), + ), + _mm_cmpeq_epi8(_mm_subs_epu8(str_vec, x20), v0), + )); + + loop { + _mm_storeu_si128(dst.cast::<__m128i>(), str_vec); + + if mask != 0 { + let cn = mask.trailing_zeros() as usize; + nb -= cn; + dst = dst.add(cn); + scratch_ptr = scratch_ptr.add(cn); + nb -= 1; + mask >>= cn + 1; + write_escape!(*(scratch_ptr), dst); + scratch_ptr = scratch_ptr.add(1); + str_vec = _mm_loadu_si128(scratch_ptr as *const __m128i); + } else { + dst = dst.add(nb); + break; + } + } + } + + core::ptr::write(dst, b'"'); + dst = dst.add(1); + + dst as usize - odst as usize + } +} diff --git a/src/serialize/writer/uuid.rs b/src/serialize/writer/uuid.rs new file mode 100644 index 00000000..ce2b1a9e --- /dev/null +++ b/src/serialize/writer/uuid.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright The Rust Project Developers (2013-2014), The Uuid Project Developers (2018) + +use crate::ffi::PyUuidRef; + +pub(crate) fn format_hyphenated(ob: PyUuidRef, dst: &mut [u8; 36]) { + const LOWER: [u8; 16] = [ + b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'a', b'b', b'c', b'd', b'e', + b'f', + ]; + const GROUPS: [(usize, usize); 5] = [(0, 8), (9, 13), (14, 18), (19, 23), (24, 36)]; + + let mut src: [u8; 16] = [0; 16]; + ob.value(&mut src); + + let mut group_idx = 0; + let mut i = 0; + while group_idx < 5 { + let (start, end) = GROUPS[group_idx]; + let mut j = start; + while j < end { + let x = src[i]; + i += 1; + + dst[j] = LOWER[(x >> 4) as usize]; + dst[j + 1] = LOWER[(x & 0x0f) as usize]; + j += 2; + } + if group_idx < 4 { + dst[end] = b'-'; + } + group_idx += 1; + } +} diff --git a/src/typeref.rs b/src/typeref.rs index 91b4e30d..a6e1b875 100644 --- a/src/typeref.rs +++ b/src/typeref.rs @@ -1,313 +1,245 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) - -use ahash::RandomState; -use once_cell::unsync::Lazy; -use pyo3_ffi::*; -use std::os::raw::c_char; -use std::ptr::NonNull; -use std::sync::Once; - -pub struct NumpyTypes { - pub array: *mut PyTypeObject, - pub float64: *mut PyTypeObject, - pub float32: *mut PyTypeObject, - pub int64: *mut PyTypeObject, - pub int32: *mut PyTypeObject, - pub int8: *mut PyTypeObject, - pub uint64: *mut PyTypeObject, - pub uint32: *mut PyTypeObject, - pub uint8: *mut PyTypeObject, - pub bool_: *mut PyTypeObject, - pub datetime64: *mut PyTypeObject, -} - -pub static mut DEFAULT: *mut PyObject = 0 as *mut PyObject; -pub static mut OPTION: *mut PyObject = 0 as *mut PyObject; - -pub static mut NONE: *mut PyObject = 0 as *mut PyObject; -pub static mut TRUE: *mut PyObject = 0 as *mut PyObject; -pub static mut FALSE: *mut PyObject = 0 as *mut PyObject; - -pub static mut BYTES_TYPE: *mut PyTypeObject = 0 as *mut PyTypeObject; -pub static mut BYTEARRAY_TYPE: *mut PyTypeObject = 0 as *mut PyTypeObject; -pub static mut MEMORYVIEW_TYPE: *mut PyTypeObject = 0 as *mut PyTypeObject; -pub static mut STR_TYPE: *mut PyTypeObject = 0 as *mut PyTypeObject; -pub static mut INT_TYPE: *mut PyTypeObject = 0 as *mut PyTypeObject; -pub static mut BOOL_TYPE: *mut PyTypeObject = 0 as *mut PyTypeObject; -pub static mut NONE_TYPE: *mut PyTypeObject = 0 as *mut PyTypeObject; -pub static mut FLOAT_TYPE: *mut PyTypeObject = 0 as *mut PyTypeObject; -pub static mut LIST_TYPE: *mut PyTypeObject = 0 as *mut PyTypeObject; -pub static mut DICT_TYPE: *mut PyTypeObject = 0 as *mut PyTypeObject; -pub static mut DATETIME_TYPE: *mut PyTypeObject = 0 as *mut PyTypeObject; -pub static mut DATE_TYPE: *mut PyTypeObject = 0 as *mut PyTypeObject; -pub static mut TIME_TYPE: *mut PyTypeObject = 0 as *mut PyTypeObject; -pub static mut TUPLE_TYPE: *mut PyTypeObject = 0 as *mut PyTypeObject; -pub static mut UUID_TYPE: *mut PyTypeObject = 0 as *mut PyTypeObject; -pub static mut ENUM_TYPE: *mut PyTypeObject = 0 as *mut PyTypeObject; - -#[cfg(Py_3_9)] -pub static mut ZONEINFO_TYPE: *mut PyTypeObject = 0 as *mut PyTypeObject; - -pub static mut NUMPY_TYPES: Lazy> = Lazy::new(|| unsafe { load_numpy_types() }); -pub static mut FIELD_TYPE: Lazy> = Lazy::new(|| unsafe { look_up_field_type() }); - -pub static mut INT_ATTR_STR: *mut PyObject = 0 as *mut PyObject; -pub static mut UTCOFFSET_METHOD_STR: *mut PyObject = 0 as *mut PyObject; -pub static mut NORMALIZE_METHOD_STR: *mut PyObject = 0 as *mut PyObject; -pub static mut CONVERT_METHOD_STR: *mut PyObject = 0 as *mut PyObject; -pub static mut EMPTY_UNICODE: *mut PyObject = 0 as *mut PyObject; -pub static mut DST_STR: *mut PyObject = 0 as *mut PyObject; -pub static mut DICT_STR: *mut PyObject = 0 as *mut PyObject; -pub static mut DATACLASS_FIELDS_STR: *mut PyObject = 0 as *mut PyObject; -pub static mut SLOTS_STR: *mut PyObject = 0 as *mut PyObject; -pub static mut FIELD_TYPE_STR: *mut PyObject = 0 as *mut PyObject; -pub static mut ARRAY_STRUCT_STR: *mut PyObject = 0 as *mut PyObject; -pub static mut DTYPE_STR: *mut PyObject = 0 as *mut PyObject; -pub static mut DESCR_STR: *mut PyObject = 0 as *mut PyObject; -pub static mut VALUE_STR: *mut PyObject = 0 as *mut PyObject; - -pub static mut STR_HASH_FUNCTION: Option = None; - -pub static mut HASH_BUILDER: Lazy = Lazy::new(|| unsafe { - RandomState::with_seeds( - VALUE_STR as u64, - DICT_TYPE as u64, - STR_TYPE as u64, - BYTES_TYPE as u64, - ) -}); -#[cfg(feature = "yyjson")] -pub const YYJSON_BUFFER_SIZE: usize = 1024 * 1024 * 8; - -#[cfg(feature = "yyjson")] -pub static mut YYJSON_ALLOC: Lazy = Lazy::new(|| unsafe { - let buffer = Box::<[u8; YYJSON_BUFFER_SIZE]>::new([0; YYJSON_BUFFER_SIZE]); - let mut alloc = crate::yyjson::yyjson_alc { - malloc: None, - realloc: None, - free: None, - ctx: std::ptr::null_mut(), - }; - crate::yyjson::yyjson_alc_pool_init( - &mut alloc, - Box::into_raw(buffer) as *mut std::os::raw::c_void, - YYJSON_BUFFER_SIZE, - ); - alloc -}); +// Copyright ijl (2020-2026), Aviram Hassan (2020-2021), Nazar Kostetskyi (2022), Ben Sully (2021) + +use core::ffi::CStr; +use core::ptr::{NonNull, null_mut}; +use once_cell::race::OnceBox; +use std::sync::OnceLock; + +use crate::ffi::{ + Py_DECREF, Py_False, Py_INCREF, Py_None, Py_True, Py_XDECREF, PyBool_Type, PyBytes_Type, + PyDict_Type, PyErr_Clear, PyErr_NewException, PyExc_TypeError, PyFloat_Type, + PyImport_ImportModule, PyList_Type, PyLong_Type, PyMapping_GetItemString, PyObject, + PyObject_GenericGetDict, PyTuple_Type, PyTypeObject, PyUnicode_InternFromString, PyUnicode_New, + PyUnicode_Type, orjson_fragmenttype_new, +}; + +pub(crate) static mut DEFAULT: *mut PyObject = null_mut(); +pub(crate) static mut OPTION: *mut PyObject = null_mut(); + +pub(crate) static mut NONE: *mut PyObject = null_mut(); +pub(crate) static mut TRUE: *mut PyObject = null_mut(); +pub(crate) static mut FALSE: *mut PyObject = null_mut(); +pub(crate) static mut EMPTY_UNICODE: *mut PyObject = null_mut(); + +pub(crate) static mut BYTES_TYPE: *mut PyTypeObject = null_mut(); +pub(crate) static mut STR_TYPE: *mut PyTypeObject = null_mut(); +pub(crate) static mut INT_TYPE: *mut PyTypeObject = null_mut(); +pub(crate) static mut BOOL_TYPE: *mut PyTypeObject = null_mut(); +pub(crate) static mut NONE_TYPE: *mut PyTypeObject = null_mut(); +pub(crate) static mut FLOAT_TYPE: *mut PyTypeObject = null_mut(); +pub(crate) static mut LIST_TYPE: *mut PyTypeObject = null_mut(); +pub(crate) static mut DICT_TYPE: *mut PyTypeObject = null_mut(); +pub(crate) static mut DATETIME_TYPE: *mut PyTypeObject = null_mut(); +pub(crate) static mut DATE_TYPE: *mut PyTypeObject = null_mut(); +pub(crate) static mut TIME_TYPE: *mut PyTypeObject = null_mut(); +pub(crate) static mut TUPLE_TYPE: *mut PyTypeObject = null_mut(); +pub(crate) static mut UUID_TYPE: *mut PyTypeObject = null_mut(); +pub(crate) static mut ENUM_TYPE: *mut PyTypeObject = null_mut(); +pub(crate) static mut FIELD_TYPE: *mut PyTypeObject = null_mut(); +pub(crate) static mut FRAGMENT_TYPE: *mut PyTypeObject = null_mut(); + +pub(crate) static mut ZONEINFO_TYPE: *mut PyTypeObject = null_mut(); + +pub(crate) static mut UTCOFFSET_METHOD_STR: *mut PyObject = null_mut(); +pub(crate) static mut NORMALIZE_METHOD_STR: *mut PyObject = null_mut(); +pub(crate) static mut CONVERT_METHOD_STR: *mut PyObject = null_mut(); +pub(crate) static mut DST_STR: *mut PyObject = null_mut(); + +pub(crate) static mut DICT_STR: *mut PyObject = null_mut(); +pub(crate) static mut DATACLASS_FIELDS_STR: *mut PyObject = null_mut(); +pub(crate) static mut SLOTS_STR: *mut PyObject = null_mut(); +pub(crate) static mut FIELD_TYPE_STR: *mut PyObject = null_mut(); +pub(crate) static mut ARRAY_STRUCT_STR: *mut PyObject = null_mut(); +pub(crate) static mut DTYPE_STR: *mut PyObject = null_mut(); +pub(crate) static mut DESCR_STR: *mut PyObject = null_mut(); +pub(crate) static mut VALUE_STR: *mut PyObject = null_mut(); +pub(crate) static mut INT_ATTR_STR: *mut PyObject = null_mut(); #[allow(non_upper_case_globals)] -pub static mut JsonEncodeError: *mut PyObject = 0 as *mut PyObject; +pub(crate) static mut JsonEncodeError: *mut PyObject = null_mut(); #[allow(non_upper_case_globals)] -pub static mut JsonDecodeError: *mut PyObject = 0 as *mut PyObject; - -static INIT: Once = Once::new(); +pub(crate) static mut JsonDecodeError: *mut PyObject = null_mut(); #[cold] -#[cfg_attr(feature = "unstable-simd", optimize(size))] -pub fn init_typerefs() { - INIT.call_once(|| unsafe { - assert!(crate::deserialize::KEY_MAP - .set(crate::deserialize::KeyMap::default()) - .is_ok()); - PyDateTime_IMPORT(); - NONE = Py_None(); - TRUE = Py_True(); - FALSE = Py_False(); - EMPTY_UNICODE = PyUnicode_New(0, 255); - STR_TYPE = (*EMPTY_UNICODE).ob_type; - STR_HASH_FUNCTION = (*((*EMPTY_UNICODE).ob_type)).tp_hash; - BYTES_TYPE = (*PyBytes_FromStringAndSize("".as_ptr() as *const c_char, 0)).ob_type; - - { - let bytearray = PyByteArray_FromStringAndSize("".as_ptr() as *const c_char, 0); - BYTEARRAY_TYPE = (*bytearray).ob_type; - - let memoryview = PyMemoryView_FromObject(bytearray); - MEMORYVIEW_TYPE = (*memoryview).ob_type; - Py_DECREF(memoryview); - Py_DECREF(bytearray); - } - - DICT_TYPE = (*PyDict_New()).ob_type; - LIST_TYPE = (*PyList_New(0)).ob_type; - TUPLE_TYPE = (*PyTuple_New(0)).ob_type; - NONE_TYPE = (*NONE).ob_type; - BOOL_TYPE = (*TRUE).ob_type; - INT_TYPE = (*PyLong_FromLongLong(0)).ob_type; - FLOAT_TYPE = (*PyFloat_FromDouble(0.0)).ob_type; - DATETIME_TYPE = look_up_datetime_type(); - DATE_TYPE = look_up_date_type(); - TIME_TYPE = look_up_time_type(); - UUID_TYPE = look_up_uuid_type(); - ENUM_TYPE = look_up_enum_type(); - - #[cfg(Py_3_9)] - { - ZONEINFO_TYPE = look_up_zoneinfo_type(); - } - - INT_ATTR_STR = PyUnicode_InternFromString("int\0".as_ptr() as *const c_char); - UTCOFFSET_METHOD_STR = PyUnicode_InternFromString("utcoffset\0".as_ptr() as *const c_char); - NORMALIZE_METHOD_STR = PyUnicode_InternFromString("normalize\0".as_ptr() as *const c_char); - CONVERT_METHOD_STR = PyUnicode_InternFromString("convert\0".as_ptr() as *const c_char); - DST_STR = PyUnicode_InternFromString("dst\0".as_ptr() as *const c_char); - DICT_STR = PyUnicode_InternFromString("__dict__\0".as_ptr() as *const c_char); - DATACLASS_FIELDS_STR = - PyUnicode_InternFromString("__dataclass_fields__\0".as_ptr() as *const c_char); - SLOTS_STR = PyUnicode_InternFromString("__slots__\0".as_ptr() as *const c_char); - FIELD_TYPE_STR = PyUnicode_InternFromString("_field_type\0".as_ptr() as *const c_char); - ARRAY_STRUCT_STR = - PyUnicode_InternFromString("__array_struct__\0".as_ptr() as *const c_char); - DTYPE_STR = PyUnicode_InternFromString("dtype\0".as_ptr() as *const c_char); - DESCR_STR = PyUnicode_InternFromString("descr\0".as_ptr() as *const c_char); - VALUE_STR = PyUnicode_InternFromString("value\0".as_ptr() as *const c_char); - DEFAULT = PyUnicode_InternFromString("default\0".as_ptr() as *const c_char); - OPTION = PyUnicode_InternFromString("option\0".as_ptr() as *const c_char); - JsonEncodeError = pyo3_ffi::PyExc_TypeError; - Py_INCREF(JsonEncodeError); - JsonDecodeError = look_up_json_exc(); - }); +#[cfg_attr(feature = "optimize", optimize(size))] +unsafe fn look_up_type_object(module_name: &CStr, member_name: &CStr) -> *mut PyTypeObject { + unsafe { + let module = PyImport_ImportModule(module_name.as_ptr()); + let module_dict = PyObject_GenericGetDict(module, null_mut()); + let ptr = PyMapping_GetItemString(module_dict, member_name.as_ptr()).cast::(); + Py_DECREF(module_dict); + Py_DECREF(module); + ptr + } } +#[cfg(not(PyPy))] #[cold] -#[cfg_attr(feature = "unstable-simd", optimize(size))] -unsafe fn look_up_json_exc() -> *mut PyObject { - let module = PyImport_ImportModule("json\0".as_ptr() as *const c_char); - let module_dict = PyObject_GenericGetDict(module, std::ptr::null_mut()); - let ptr = PyMapping_GetItemString(module_dict, "JSONDecodeError\0".as_ptr() as *const c_char) - as *mut PyObject; - let res = pyo3_ffi::PyErr_NewException( - "orjson.JSONDecodeError\0".as_ptr() as *const c_char, - ptr, - std::ptr::null_mut(), - ); - Py_DECREF(ptr); - Py_DECREF(module_dict); - Py_DECREF(module); - Py_INCREF(res); - res +#[cfg_attr(feature = "optimize", optimize(size))] +unsafe fn look_up_datetime() { + unsafe { + crate::ffi::PyDateTime_IMPORT(); + let datetime_capsule = crate::ffi::PyCapsule_Import(c"datetime.datetime_CAPI".as_ptr(), 1) + .cast::(); + debug_assert!(!datetime_capsule.is_null()); + + DATETIME_TYPE = (*datetime_capsule).DateTimeType; + DATE_TYPE = (*datetime_capsule).DateType; + TIME_TYPE = (*datetime_capsule).TimeType; + ZONEINFO_TYPE = (*datetime_capsule).TZInfoType; + } } +#[cfg(PyPy)] #[cold] -#[cfg_attr(feature = "unstable-simd", optimize(size))] -unsafe fn look_up_numpy_type(numpy_module: *mut PyObject, np_type: &str) -> *mut PyTypeObject { - let mod_dict = PyObject_GenericGetDict(numpy_module, std::ptr::null_mut()); - let ptr = PyMapping_GetItemString(mod_dict, np_type.as_ptr() as *const c_char); - Py_XDECREF(ptr); - Py_XDECREF(mod_dict); - ptr as *mut PyTypeObject +#[cfg_attr(feature = "optimize", optimize(size))] +unsafe fn look_up_datetime() { + unsafe { + DATETIME_TYPE = look_up_type_object(c"datetime", c"datetime"); + DATE_TYPE = look_up_type_object(c"datetime", c"date"); + TIME_TYPE = look_up_type_object(c"datetime", c"time"); + ZONEINFO_TYPE = look_up_type_object(c"zoneinfo", c"ZoneInfo"); + } } -#[cold] -#[cfg_attr(feature = "unstable-simd", optimize(size))] -unsafe fn load_numpy_types() -> Option { - let numpy = PyImport_ImportModule("numpy\0".as_ptr() as *const c_char); - if numpy.is_null() { - PyErr_Clear(); - return None; - } +static INIT: OnceLock = OnceLock::new(); - let types = Some(NumpyTypes { - array: look_up_numpy_type(numpy, "ndarray\0"), - float32: look_up_numpy_type(numpy, "float32\0"), - float64: look_up_numpy_type(numpy, "float64\0"), - int8: look_up_numpy_type(numpy, "int8\0"), - int32: look_up_numpy_type(numpy, "int32\0"), - int64: look_up_numpy_type(numpy, "int64\0"), - uint32: look_up_numpy_type(numpy, "uint32\0"), - uint64: look_up_numpy_type(numpy, "uint64\0"), - uint8: look_up_numpy_type(numpy, "uint8\0"), - bool_: look_up_numpy_type(numpy, "bool_\0"), - datetime64: look_up_numpy_type(numpy, "datetime64\0"), - }); - Py_XDECREF(numpy); - types +pub(crate) fn init_typerefs() { + INIT.get_or_init(_init_typerefs_impl); } #[cold] -#[cfg_attr(feature = "unstable-simd", optimize(size))] -unsafe fn look_up_field_type() -> NonNull { - let module = PyImport_ImportModule("dataclasses\0".as_ptr() as *const c_char); - let module_dict = PyObject_GenericGetDict(module, std::ptr::null_mut()); - let ptr = PyMapping_GetItemString(module_dict, "_FIELD\0".as_ptr() as *const c_char) - as *mut PyTypeObject; - Py_DECREF(module_dict); - Py_DECREF(module); - NonNull::new_unchecked(ptr as *mut PyObject) -} +#[cfg_attr(feature = "optimize", optimize(size))] +fn _init_typerefs_impl() -> bool { + unsafe { + debug_assert!(crate::opt::MAX_OPT < i32::from(u16::MAX)); -#[cold] -#[cfg_attr(feature = "unstable-simd", optimize(size))] -unsafe fn look_up_enum_type() -> *mut PyTypeObject { - let module = PyImport_ImportModule("enum\0".as_ptr() as *const c_char); - let module_dict = PyObject_GenericGetDict(module, std::ptr::null_mut()); - let ptr = PyMapping_GetItemString(module_dict, "EnumMeta\0".as_ptr() as *const c_char) - as *mut PyTypeObject; - Py_DECREF(module_dict); - Py_DECREF(module); - ptr -} + #[cfg(not(Py_GIL_DISABLED))] + assert!( + crate::deserialize::KEY_MAP + .set(crate::deserialize::KeyMap::default()) + .is_ok() + ); -#[cold] -#[cfg_attr(feature = "unstable-simd", optimize(size))] -unsafe fn look_up_uuid_type() -> *mut PyTypeObject { - let uuid_mod = PyImport_ImportModule("uuid\0".as_ptr() as *const c_char); - let uuid_mod_dict = PyObject_GenericGetDict(uuid_mod, std::ptr::null_mut()); - let uuid = PyMapping_GetItemString(uuid_mod_dict, "NAMESPACE_DNS\0".as_ptr() as *const c_char); - let ptr = (*uuid).ob_type; - Py_DECREF(uuid); - Py_DECREF(uuid_mod_dict); - Py_DECREF(uuid_mod); - ptr -} + crate::serialize::set_str_formatter_fn(); + crate::ffi::set_str_create_fn(); -#[cold] -#[cfg_attr(feature = "unstable-simd", optimize(size))] -unsafe fn look_up_datetime_type() -> *mut PyTypeObject { - let datetime = ((*PyDateTimeAPI()).DateTime_FromDateAndTime)( - 1970, - 1, - 1, - 0, - 0, - 0, - 0, - NONE, - (*(PyDateTimeAPI())).DateTimeType, - ); - let ptr = (*datetime).ob_type; - Py_DECREF(datetime); - ptr + NONE = Py_None(); + TRUE = Py_True(); + FALSE = Py_False(); + EMPTY_UNICODE = PyUnicode_New(0, 255); + + STR_TYPE = &raw mut PyUnicode_Type; + BYTES_TYPE = &raw mut PyBytes_Type; + DICT_TYPE = &raw mut PyDict_Type; + LIST_TYPE = &raw mut PyList_Type; + TUPLE_TYPE = &raw mut PyTuple_Type; + NONE_TYPE = crate::ffi::PyObject_Type(NONE); + BOOL_TYPE = &raw mut PyBool_Type; + INT_TYPE = &raw mut PyLong_Type; + FLOAT_TYPE = &raw mut PyFloat_Type; + + look_up_datetime(); + + UUID_TYPE = look_up_type_object(c"uuid", c"UUID"); + ENUM_TYPE = look_up_type_object(c"enum", c"EnumMeta"); + FIELD_TYPE = look_up_type_object(c"dataclasses", c"_FIELD"); + + FRAGMENT_TYPE = orjson_fragmenttype_new(); + + INT_ATTR_STR = PyUnicode_InternFromString(c"int".as_ptr()); + UTCOFFSET_METHOD_STR = PyUnicode_InternFromString(c"utcoffset".as_ptr()); + NORMALIZE_METHOD_STR = PyUnicode_InternFromString(c"normalize".as_ptr()); + CONVERT_METHOD_STR = PyUnicode_InternFromString(c"convert".as_ptr()); + DST_STR = PyUnicode_InternFromString(c"dst".as_ptr()); + DICT_STR = PyUnicode_InternFromString(c"__dict__".as_ptr()); + DATACLASS_FIELDS_STR = PyUnicode_InternFromString(c"__dataclass_fields__".as_ptr()); + SLOTS_STR = PyUnicode_InternFromString(c"__slots__".as_ptr()); + FIELD_TYPE_STR = PyUnicode_InternFromString(c"_field_type".as_ptr()); + ARRAY_STRUCT_STR = PyUnicode_InternFromString(c"__array_struct__".as_ptr()); + DTYPE_STR = PyUnicode_InternFromString(c"dtype".as_ptr()); + DESCR_STR = PyUnicode_InternFromString(c"descr".as_ptr()); + VALUE_STR = PyUnicode_InternFromString(c"value".as_ptr()); + DEFAULT = PyUnicode_InternFromString(c"default".as_ptr()); + OPTION = PyUnicode_InternFromString(c"option".as_ptr()); + + JsonEncodeError = PyExc_TypeError; + Py_INCREF(JsonEncodeError); + let json_jsondecodeerror = + look_up_type_object(c"json", c"JSONDecodeError").cast::(); + debug_assert!(!json_jsondecodeerror.is_null()); + JsonDecodeError = PyErr_NewException( + c"orjson.JSONDecodeError".as_ptr(), + json_jsondecodeerror, + null_mut(), + ); + debug_assert!(!JsonDecodeError.is_null()); + Py_XDECREF(json_jsondecodeerror); + }; + true } -#[cold] -#[cfg_attr(feature = "unstable-simd", optimize(size))] -unsafe fn look_up_date_type() -> *mut PyTypeObject { - let date = ((*PyDateTimeAPI()).Date_FromDate)(1, 1, 1, (*(PyDateTimeAPI())).DateType); - let ptr = (*date).ob_type; - Py_DECREF(date); - ptr +pub(crate) struct NumpyTypes { + pub array: *mut PyTypeObject, + pub float64: *mut PyTypeObject, + pub float32: *mut PyTypeObject, + pub float16: *mut PyTypeObject, + pub int64: *mut PyTypeObject, + pub int32: *mut PyTypeObject, + pub int16: *mut PyTypeObject, + pub int8: *mut PyTypeObject, + pub uint64: *mut PyTypeObject, + pub uint32: *mut PyTypeObject, + pub uint16: *mut PyTypeObject, + pub uint8: *mut PyTypeObject, + pub bool_: *mut PyTypeObject, + pub datetime64: *mut PyTypeObject, } +pub(crate) static mut NUMPY_TYPES: OnceBox>> = OnceBox::new(); + #[cold] -#[cfg_attr(feature = "unstable-simd", optimize(size))] -unsafe fn look_up_time_type() -> *mut PyTypeObject { - let time = ((*PyDateTimeAPI()).Time_FromTime)(0, 0, 0, 0, NONE, (*(PyDateTimeAPI())).TimeType); - let ptr = (*time).ob_type; - Py_DECREF(time); - ptr +#[cfg_attr(feature = "optimize", optimize(size))] +unsafe fn look_up_numpy_type( + numpy_module_dict: *mut PyObject, + np_type: &CStr, +) -> *mut PyTypeObject { + unsafe { + let ptr = PyMapping_GetItemString(numpy_module_dict, np_type.as_ptr()); + Py_XDECREF(ptr); + ptr.cast::() + } } -#[cfg(Py_3_9)] #[cold] -#[cfg_attr(feature = "unstable-simd", optimize(size))] -unsafe fn look_up_zoneinfo_type() -> *mut PyTypeObject { - let module = PyImport_ImportModule("zoneinfo\0".as_ptr() as *const c_char); - let module_dict = PyObject_GenericGetDict(module, std::ptr::null_mut()); - let ptr = PyMapping_GetItemString(module_dict, "ZoneInfo\0".as_ptr() as *const c_char) - as *mut PyTypeObject; - Py_DECREF(module_dict); - Py_DECREF(module); - ptr +#[cfg_attr(feature = "optimize", optimize(size))] +pub(crate) fn load_numpy_types() -> Box>> { + unsafe { + let numpy = PyImport_ImportModule(c"numpy".as_ptr()); + if numpy.is_null() { + PyErr_Clear(); + return Box::new(None); + } + let numpy_module_dict = PyObject_GenericGetDict(numpy, null_mut()); + let types = Box::new(NumpyTypes { + array: look_up_numpy_type(numpy_module_dict, c"ndarray"), + float16: look_up_numpy_type(numpy_module_dict, c"half"), + float32: look_up_numpy_type(numpy_module_dict, c"float32"), + float64: look_up_numpy_type(numpy_module_dict, c"float64"), + int8: look_up_numpy_type(numpy_module_dict, c"int8"), + int16: look_up_numpy_type(numpy_module_dict, c"int16"), + int32: look_up_numpy_type(numpy_module_dict, c"int32"), + int64: look_up_numpy_type(numpy_module_dict, c"int64"), + uint16: look_up_numpy_type(numpy_module_dict, c"uint16"), + uint32: look_up_numpy_type(numpy_module_dict, c"uint32"), + uint64: look_up_numpy_type(numpy_module_dict, c"uint64"), + uint8: look_up_numpy_type(numpy_module_dict, c"uint8"), + bool_: look_up_numpy_type(numpy_module_dict, c"bool_"), + datetime64: look_up_numpy_type(numpy_module_dict, c"datetime64"), + }); + Py_XDECREF(numpy_module_dict); + Py_XDECREF(numpy); + Box::new(Some(nonnull!(Box::::into_raw(types)))) + } } diff --git a/src/unicode.rs b/src/unicode.rs deleted file mode 100644 index 49630015..00000000 --- a/src/unicode.rs +++ /dev/null @@ -1,154 +0,0 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -use crate::typeref::EMPTY_UNICODE; -use crate::typeref::STR_HASH_FUNCTION; -use pyo3_ffi::*; -use std::os::raw::c_char; - -// see unicodeobject.h for documentation -// re: python3.12 changes, https://www.python.org/dev/peps/pep-0623/ - -#[repr(C)] -pub struct PyASCIIObject { - pub ob_base: PyObject, - pub length: Py_ssize_t, - pub hash: Py_hash_t, - pub state: u32, - #[cfg(not(Py_3_12))] - pub wstr: *mut c_char, -} - -#[repr(C)] -pub struct PyCompactUnicodeObject { - pub ob_base: PyASCIIObject, - pub utf8_length: Py_ssize_t, - pub utf8: *mut c_char, - #[cfg(not(Py_3_12))] - pub wstr_length: Py_ssize_t, -} - -#[cfg(not(Py_3_12))] -const STATE_ASCII: u32 = 0b00000000000000000000000001000000; -#[cfg(not(Py_3_12))] -const STATE_COMPACT: u32 = 0b00000000000000000000000000100000; - -#[cfg(Py_3_12)] -const STATE_ASCII: u32 = 0b00000000000000000000000000100000; - -#[cfg(Py_3_12)] -const STATE_COMPACT: u32 = 0b00000000000000000000000000010000; - -const STATE_COMPACT_ASCII: u32 = STATE_COMPACT | STATE_ASCII; - -fn is_four_byte(buf: &str) -> bool { - let mut ret = false; - for &each in buf.as_bytes() { - ret |= each >= 240; - } - ret -} - -enum PyUnicodeKind { - Ascii, - OneByte, - TwoByte, - FourByte, -} - -fn find_str_kind(buf: &str, num_chars: usize) -> PyUnicodeKind { - if buf.len() == num_chars { - PyUnicodeKind::Ascii - } else if is_four_byte(buf) { - PyUnicodeKind::FourByte - } else if encoding_rs::mem::is_str_latin1(buf) { - PyUnicodeKind::OneByte - } else { - PyUnicodeKind::TwoByte - } -} - -pub fn unicode_from_str(buf: &str) -> *mut pyo3_ffi::PyObject { - let len = buf.len(); - if unlikely!(len == 0) { - ffi!(Py_INCREF(EMPTY_UNICODE)); - unsafe { EMPTY_UNICODE } - } else { - let num_chars = bytecount::num_chars(buf.as_bytes()) as isize; - match find_str_kind(buf, num_chars as usize) { - PyUnicodeKind::Ascii => unsafe { - let ptr = ffi!(PyUnicode_New(len as isize, 127)); - let data_ptr = ptr.cast::().offset(1) as *mut u8; - core::ptr::copy_nonoverlapping(buf.as_ptr(), data_ptr, len); - core::ptr::write(data_ptr.add(len), 0); - ptr - }, - PyUnicodeKind::OneByte => unsafe { - PyUnicode_DecodeUTF8( - buf.as_bytes().as_ptr() as *const c_char, - buf.as_bytes().len() as isize, - "ignore\0".as_ptr() as *const c_char, - ) - }, - PyUnicodeKind::TwoByte => unsafe { - let ptr = ffi!(PyUnicode_New(num_chars, 65535)); - (*ptr.cast::()).length = num_chars; - let mut data_ptr = ptr.cast::().offset(1) as *mut u16; - for each in buf.chars() { - core::ptr::write(data_ptr, each as u16); - data_ptr = data_ptr.offset(1); - } - core::ptr::write(data_ptr, 0); - ptr - }, - PyUnicodeKind::FourByte => unsafe { - let ptr = ffi!(PyUnicode_New(num_chars, 1114111)); - (*ptr.cast::()).length = num_chars; - let mut data_ptr = ptr.cast::().offset(1) as *mut u32; - for each in buf.chars() { - core::ptr::write(data_ptr, each as u32); - data_ptr = data_ptr.offset(1); - } - core::ptr::write(data_ptr, 0); - ptr - }, - } - } -} - -#[inline] -pub fn hash_str(op: *mut PyObject) -> Py_hash_t { - unsafe { - (*op.cast::()).hash = STR_HASH_FUNCTION.unwrap()(op); - (*op.cast::()).hash - } -} - -#[inline(never)] -pub fn unicode_to_str_via_ffi(op: *mut PyObject) -> Option<&'static str> { - let mut str_size: pyo3_ffi::Py_ssize_t = 0; - let ptr = ffi!(PyUnicode_AsUTF8AndSize(op, &mut str_size)) as *const u8; - if unlikely!(ptr.is_null()) { - None - } else { - Some(str_from_slice!(ptr, str_size as usize)) - } -} - -#[inline(always)] -pub fn unicode_to_str(op: *mut PyObject) -> Option<&'static str> { - unsafe { - if (*op.cast::()).state & STATE_COMPACT_ASCII == STATE_COMPACT_ASCII { - let ptr = op.cast::().offset(1) as *const u8; - let len = (*op.cast::()).length as usize; - Some(str_from_slice!(ptr, len)) - } else if (*op.cast::()).state & STATE_COMPACT == STATE_COMPACT - && !(*op.cast::()).utf8.is_null() - { - let ptr = (*op.cast::()).utf8 as *const u8; - let len = (*op.cast::()).utf8_length as usize; - Some(str_from_slice!(ptr, len)) - } else { - unicode_to_str_via_ffi(op) - } - } -} diff --git a/src/util.rs b/src/util.rs index 859508c6..bfeccbfe 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,4 +1,7 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright ijl (2019-2026), Ben Sully (2021), Marc Mueller (2023) + +pub(crate) const INVALID_STR: &str = "str is not valid UTF-8: surrogates not allowed"; macro_rules! is_type { ($obj_ptr:expr, $type_ptr:expr) => { @@ -6,9 +9,27 @@ macro_rules! is_type { }; } -macro_rules! ob_type { - ($obj:expr) => { - unsafe { (*$obj).ob_type } +macro_rules! is_class_by_type { + ($ob_type:expr, $type_ptr:ident) => { + unsafe { $ob_type == $type_ptr } + }; +} + +macro_rules! is_subclass_by_flag { + ($tp_flags:expr, $flag:ident) => { + unsafe { (($tp_flags & crate::ffi::$flag) != 0) } + }; +} + +macro_rules! is_subclass_by_type { + ($ob_type:expr, $type:ident) => { + unsafe { + (*($ob_type.cast::())) + .ob_base + .ob_base + .ob_type + == $type + } }; } @@ -18,83 +39,207 @@ macro_rules! err { }; } -#[cfg(feature = "unstable-simd")] -macro_rules! unlikely { - ($exp:expr) => { - core::intrinsics::unlikely($exp) +macro_rules! opt_enabled { + ($var:expr, $flag:expr) => { + $var & $flag != 0 }; } -#[cfg(not(feature = "unstable-simd"))] -macro_rules! unlikely { - ($exp:expr) => { - $exp +macro_rules! opt_disabled { + ($var:expr, $flag:expr) => { + $var & $flag == 0 + }; +} + +macro_rules! cold_path { + () => { + core::hint::cold_path(); }; } macro_rules! nonnull { ($exp:expr) => { - unsafe { std::ptr::NonNull::new_unchecked($exp) } + unsafe { core::ptr::NonNull::new_unchecked($exp) } }; } macro_rules! str_from_slice { ($ptr:expr, $size:expr) => { - unsafe { std::str::from_utf8_unchecked(std::slice::from_raw_parts($ptr, $size as usize)) } + unsafe { core::str::from_utf8_unchecked(core::slice::from_raw_parts($ptr, $size as usize)) } + }; +} + +#[cfg(all(Py_3_12, not(Py_GIL_DISABLED)))] +macro_rules! reverse_pydict_incref { + ($op:expr) => { + unsafe { + if crate::ffi::_Py_IsImmortal($op) == 0 { + (*$op).ob_refcnt.ob_refcnt -= 1; + } + } + }; +} + +#[cfg(Py_GIL_DISABLED)] +macro_rules! reverse_pydict_incref { + ($op:expr) => { + unsafe { crate::ffi::Py_DECREF($op) } + }; +} + +#[cfg(not(Py_3_12))] +macro_rules! reverse_pydict_incref { + ($op:expr) => { + unsafe { + (*$op).ob_refcnt -= 1; + } }; } macro_rules! ffi { ($fn:ident()) => { - unsafe { pyo3_ffi::$fn() } + unsafe { crate::ffi::$fn() } }; ($fn:ident($obj1:expr)) => { - unsafe { pyo3_ffi::$fn($obj1) } + unsafe { crate::ffi::$fn($obj1) } }; ($fn:ident($obj1:expr, $obj2:expr)) => { - unsafe { pyo3_ffi::$fn($obj1, $obj2) } + unsafe { crate::ffi::$fn($obj1, $obj2) } }; ($fn:ident($obj1:expr, $obj2:expr, $obj3:expr)) => { - unsafe { pyo3_ffi::$fn($obj1, $obj2, $obj3) } + unsafe { crate::ffi::$fn($obj1, $obj2, $obj3) } }; ($fn:ident($obj1:expr, $obj2:expr, $obj3:expr, $obj4:expr)) => { - unsafe { pyo3_ffi::$fn($obj1, $obj2, $obj3, $obj4) } + unsafe { crate::ffi::$fn($obj1, $obj2, $obj3, $obj4) } }; } -#[cfg(Py_3_9)] -macro_rules! call_method { +#[cfg(all(CPython, Py_3_13))] +macro_rules! pydict_contains { ($obj1:expr, $obj2:expr) => { - unsafe { pyo3_ffi::PyObject_CallMethodNoArgs($obj1, $obj2) } - }; - ($obj1:expr, $obj2:expr, $obj3:expr) => { - unsafe { pyo3_ffi::PyObject_CallMethodOneArg($obj1, $obj2, $obj3) } + unsafe { crate::ffi::PyDict_Contains(crate::ffi::PyType_GetDict($obj1), $obj2) == 1 } }; } -#[cfg(not(Py_3_9))] -macro_rules! call_method { +#[cfg(all(CPython, Py_3_12, not(Py_3_13)))] +macro_rules! pydict_contains { ($obj1:expr, $obj2:expr) => { unsafe { - pyo3_ffi::PyObject_CallMethodObjArgs( - $obj1, + debug_assert!((*$obj2.cast::()).hash != -1); + crate::ffi::_PyDict_Contains_KnownHash( + crate::ffi::PyType_GetDict($obj1), $obj2, - std::ptr::null_mut() as *mut pyo3_ffi::PyObject, - ) + (*$obj2.cast::()).hash, + ) == 1 } }; - ($obj1:expr, $obj2:expr, $obj3:expr) => { +} + +#[cfg(all(CPython, not(Py_3_12)))] +macro_rules! pydict_contains { + ($obj1:expr, $obj2:expr) => { unsafe { - pyo3_ffi::PyObject_CallMethodObjArgs( - $obj1, + debug_assert!((*$obj2.cast::()).hash != -1); + crate::ffi::_PyDict_Contains_KnownHash( + (*$obj1).tp_dict, $obj2, - $obj3, - std::ptr::null_mut() as *mut pyo3_ffi::PyObject, - ) + (*$obj2.cast::()).hash, + ) == 1 + } + }; +} + +#[cfg(not(CPython))] +macro_rules! pydict_contains { + ($obj1:expr, $obj2:expr) => { + unsafe { crate::ffi::PyDict_Contains((*$obj1).tp_dict, $obj2) == 1 } + }; +} + +#[cfg(Py_3_12)] +macro_rules! use_immortal { + ($op:expr) => { + unsafe { $op } + }; +} + +#[cfg(not(Py_3_12))] +macro_rules! use_immortal { + ($op:expr) => { + unsafe { + unsafe { crate::ffi::Py_INCREF($op) }; + $op + } + }; +} + +macro_rules! assume { + ($expr:expr) => { + debug_assert!($expr); + unsafe { + core::hint::assert_unchecked($expr); + }; + }; +} + +macro_rules! unreachable_unchecked { + () => { + unsafe { core::hint::unreachable_unchecked() } + }; +} + +#[inline(always)] +#[allow(clippy::cast_possible_wrap)] +pub(crate) fn usize_to_isize(val: usize) -> isize { + debug_assert!(val < (isize::MAX as usize)); + val as isize +} + +#[inline(always)] +pub(crate) fn isize_to_usize(val: isize) -> usize { + debug_assert!(val >= 0); + val.cast_unsigned() +} + +macro_rules! write_double_digit { + ($buf:ident, $value:expr) => { + if $value < 10 { + $buf.put_u8(b'0'); + } + crate::serialize::writer::write_integer_u32($buf, $value); + }; +} + +macro_rules! write_triple_digit { + ($buf:ident, $value:expr) => { + if $value < 100 { + $buf.put_u8(b'0'); + } + if $value < 10 { + $buf.put_u8(b'0'); + } + crate::serialize::writer::write_integer_u32($buf, $value); + }; +} + +macro_rules! write_microsecond { + ($buf:ident, $microsecond:expr) => { + unsafe { + if $microsecond != 0 { + $buf.put_u8(b'.'); + match $microsecond { + 0..=9 => $buf.put_slice(b"00000"), + 10..=99 => $buf.put_slice(b"0000"), + 100..=999 => $buf.put_slice(b"000"), + 1000..=9999 => $buf.put_slice(b"00"), + _ => {} + } + crate::serialize::writer::write_integer_u32($buf, $microsecond); + } } }; } diff --git a/src/yyjson.rs b/src/yyjson.rs deleted file mode 100644 index 0931ae31..00000000 --- a/src/yyjson.rs +++ /dev/null @@ -1,94 +0,0 @@ -pub type __uint32_t = ::std::os::raw::c_uint; -pub type __int64_t = ::std::os::raw::c_long; -pub type __uint64_t = ::std::os::raw::c_ulong; -#[repr(C)] -#[derive(Copy, Clone)] -pub struct yyjson_alc { - pub malloc: ::std::option::Option< - unsafe extern "C" fn( - ctx: *mut ::std::os::raw::c_void, - size: usize, - ) -> *mut ::std::os::raw::c_void, - >, - pub realloc: ::std::option::Option< - unsafe extern "C" fn( - ctx: *mut ::std::os::raw::c_void, - ptr: *mut ::std::os::raw::c_void, - size: usize, - ) -> *mut ::std::os::raw::c_void, - >, - pub free: ::std::option::Option< - unsafe extern "C" fn(ctx: *mut ::std::os::raw::c_void, ptr: *mut ::std::os::raw::c_void), - >, - pub ctx: *mut ::std::os::raw::c_void, -} -extern "C" { - pub fn yyjson_alc_pool_init( - alc: *mut yyjson_alc, - buf: *mut ::std::os::raw::c_void, - size: usize, - ) -> bool; -} -pub type yyjson_read_flag = u32; -pub const YYJSON_READ_NOFLAG: yyjson_read_flag = 0; -pub type yyjson_read_code = u32; -pub const YYJSON_READ_SUCCESS: yyjson_read_code = 0; -#[repr(C)] -#[derive(Copy, Clone)] -pub struct yyjson_read_err { - pub code: yyjson_read_code, - pub msg: *const ::std::os::raw::c_char, - pub pos: usize, -} -extern "C" { - pub fn yyjson_read_opts( - dat: *mut ::std::os::raw::c_char, - len: usize, - flg: yyjson_read_flag, - alc: *const yyjson_alc, - err: *mut yyjson_read_err, - ) -> *mut yyjson_doc; -} -extern "C" { - pub fn yyjson_doc_free(doc: *mut yyjson_doc); -} -#[repr(C)] -#[derive(Copy, Clone)] -pub union yyjson_val_uni { - pub u64_: u64, - pub i64_: i64, - pub f64_: f64, - pub str_: *const ::std::os::raw::c_char, - pub ptr: *mut ::std::os::raw::c_void, - pub ofs: usize, -} -#[repr(C)] -#[derive(Copy, Clone)] -pub struct yyjson_val { - pub tag: u64, - pub uni: yyjson_val_uni, -} -#[repr(C)] -#[derive(Copy, Clone)] -pub struct yyjson_doc { - pub root: *mut yyjson_val, - pub alc: yyjson_alc, - pub dat_read: usize, - pub val_read: usize, - pub str_pool: *mut ::std::os::raw::c_char, -} -#[repr(C)] -#[derive(Copy, Clone)] -pub struct yyjson_arr_iter { - pub idx: usize, - pub max: usize, - pub cur: *mut yyjson_val, -} -#[repr(C)] -#[derive(Copy, Clone)] -pub struct yyjson_obj_iter { - pub idx: usize, - pub max: usize, - pub cur: *mut yyjson_val, - pub obj: *mut yyjson_val, -} diff --git a/test/requirements.txt b/test/requirements.txt index faa352c1..e1324152 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,8 +1,7 @@ -arrow -numpy;platform_machine=="x86_64" and python_version<"3.11" -pendulum;sys_platform=="linux" and platform_machine=="x86_64" -psutil;sys_platform=="linux" or sys_platform == "macos" +faker +numpy;(platform_machine=="x86_64" or (platform_machine=="aarch64" and sys_platform == "linux")) and python_version<"3.15" and implementation_name=="cpython" +pendulum;sys_platform=="linux" and platform_machine=="x86_64" and python_version<"3.15" and implementation_name=="cpython" +psutil;(sys_platform=="linux" or sys_platform == "macos") and platform_machine=="x86_64" and python_version<"3.14" and implementation_name=="cpython" pytest +python-dateutil >=2,<3;python_version<"3.15" and implementation_name=="cpython" pytz -typing_extensions;python_version<"3.8" -xxhash==1.4.3;sys_platform=="linux" and python_version<"3.9" # creates non-compact ASCII for test_str_ascii diff --git a/test/test_api.py b/test/test_api.py index 0e5a6f14..d455c3d2 100644 --- a/test/test_api.py +++ b/test/test_api.py @@ -1,147 +1,225 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2018-2025), hauntsaninja (2020) import datetime import inspect import json -import unittest +import re + +import pytest import orjson SIMPLE_TYPES = (1, 1.0, -1, None, "str", True, False) +LOADS_RECURSION_LIMIT = 1024 + def default(obj): return str(obj) -class ApiTests(unittest.TestCase): +class TestApi: def test_loads_trailing(self): """ loads() handles trailing whitespace """ - self.assertEqual(orjson.loads("{}\n\t "), {}) + assert orjson.loads("{}\n\t ") == {} def test_loads_trailing_invalid(self): """ loads() handles trailing invalid """ - self.assertRaises(orjson.JSONDecodeError, orjson.loads, "{}\n\t a") + pytest.raises(orjson.JSONDecodeError, orjson.loads, "{}\n\t a") def test_simple_json(self): """ dumps() equivalent to json on simple types """ for obj in SIMPLE_TYPES: - self.assertEqual(orjson.dumps(obj), json.dumps(obj).encode("utf-8")) + assert orjson.dumps(obj) == json.dumps(obj).encode("utf-8") def test_simple_round_trip(self): """ dumps(), loads() round trip on simple types """ for obj in SIMPLE_TYPES: - self.assertEqual(orjson.loads(orjson.dumps(obj)), obj) + assert orjson.loads(orjson.dumps(obj)) == obj def test_loads_type(self): """ loads() invalid type """ - for val in (1, 3.14, [], {}, None): - self.assertRaises(orjson.JSONDecodeError, orjson.loads, val) + for val in (1, 3.14, [], {}, None): # type: ignore + pytest.raises(orjson.JSONDecodeError, orjson.loads, val) + + def test_loads_recursion_partial(self): + """ + loads() recursion limit partial + """ + pytest.raises(orjson.JSONDecodeError, orjson.loads, "[" * (1024 * 1024)) + + def test_loads_recursion_valid_limit_array(self): + """ + loads() recursion limit at limit array + """ + n = LOADS_RECURSION_LIMIT + 1 + value = b"[" * n + b"]" * n + pytest.raises(orjson.JSONDecodeError, orjson.loads, value) + + def test_loads_recursion_valid_limit_object(self): + """ + loads() recursion limit at limit object + """ + n = LOADS_RECURSION_LIMIT + value = b'{"key":' * n + b'{"key":true}' + b"}" * n + pytest.raises(orjson.JSONDecodeError, orjson.loads, value) + + def test_loads_recursion_valid_limit_mixed(self): + """ + loads() recursion limit at limit mixed + """ + n = LOADS_RECURSION_LIMIT + value = b"".join((b"[", b'{"key":' * n, b'{"key":true}' + b"}" * n, b"]")) + pytest.raises(orjson.JSONDecodeError, orjson.loads, value) + + def test_loads_recursion_valid_excessive_array(self): + """ + loads() recursion limit excessively high value + """ + n = 10000000 + value = b"[" * n + b"]" * n + pytest.raises(orjson.JSONDecodeError, orjson.loads, value) + + def test_loads_recursion_valid_limit_array_pretty(self): + """ + loads() recursion limit at limit array pretty + """ + n = LOADS_RECURSION_LIMIT + 1 + value = b"[\n " * n + b"]" * n + pytest.raises(orjson.JSONDecodeError, orjson.loads, value) + + def test_loads_recursion_valid_limit_object_pretty(self): + """ + loads() recursion limit at limit object pretty + """ + n = LOADS_RECURSION_LIMIT + value = b'{\n "key":' * n + b'{"key":true}' + b"}" * n + pytest.raises(orjson.JSONDecodeError, orjson.loads, value) - def test_loads_recursion(self): + def test_loads_recursion_valid_limit_mixed_pretty(self): """ - loads() recursion limit + loads() recursion limit at limit mixed pretty """ - self.assertRaises(orjson.JSONDecodeError, orjson.loads, "[" * (1024 * 1024)) + n = LOADS_RECURSION_LIMIT + value = b'[\n {"key":' * n + b'{"key":true}' + b"}" * n + b"]" + pytest.raises(orjson.JSONDecodeError, orjson.loads, value) + + def test_loads_recursion_valid_excessive_array_pretty(self): + """ + loads() recursion limit excessively high value pretty + """ + n = 10000000 + value = b"[\n " * n + b"]" * n + pytest.raises(orjson.JSONDecodeError, orjson.loads, value) def test_version(self): """ __version__ """ - self.assertRegex(orjson.__version__, r"^\d+\.\d+(\.\d+)?$") + assert re.match(r"^\d+\.\d+(\.\d+)?$", orjson.__version__) def test_valueerror(self): """ orjson.JSONDecodeError is a subclass of ValueError """ - self.assertRaises(orjson.JSONDecodeError, orjson.loads, "{") - self.assertRaises(ValueError, orjson.loads, "{") + pytest.raises(orjson.JSONDecodeError, orjson.loads, "{") + pytest.raises(ValueError, orjson.loads, "{") + + def test_optional_none(self): + """ + dumps() option, default None + """ + assert orjson.dumps([], option=None) == b"[]" + assert orjson.dumps([], default=None) == b"[]" + assert orjson.dumps([], option=None, default=None) == b"[]" + assert orjson.dumps([], None, None) == b"[]" def test_option_not_int(self): """ dumps() option not int or None """ - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(True, option=True) def test_option_invalid_int(self): """ dumps() option invalid 64-bit number """ - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(True, option=9223372036854775809) def test_option_range_low(self): """ dumps() option out of range low """ - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(True, option=-1) def test_option_range_high(self): """ dumps() option out of range high """ - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(True, option=1 << 12) def test_opts_multiple(self): """ dumps() multiple option """ - self.assertEqual( + assert ( orjson.dumps( [1, datetime.datetime(2000, 1, 1, 2, 3, 4)], option=orjson.OPT_STRICT_INTEGER | orjson.OPT_NAIVE_UTC, - ), - b'[1,"2000-01-01T02:03:04+00:00"]', + ) + == b'[1,"2000-01-01T02:03:04+00:00"]' ) def test_default_positional(self): """ dumps() positional arg """ - with self.assertRaises(TypeError): - orjson.dumps(__obj={}) - with self.assertRaises(TypeError): - orjson.dumps(zxc={}) + with pytest.raises(TypeError): + orjson.dumps(__obj={}) # type: ignore + with pytest.raises(TypeError): + orjson.dumps(zxc={}) # type: ignore def test_default_unknown_kwarg(self): """ dumps() unknown kwarg """ - with self.assertRaises(TypeError): - orjson.dumps({}, zxc=default) + with pytest.raises(TypeError): + orjson.dumps({}, zxc=default) # type: ignore def test_default_empty_kwarg(self): """ dumps() empty kwarg """ - self.assertEqual(orjson.dumps(None, **{}), b"null") + assert orjson.dumps(None) == b"null" def test_default_twice(self): """ dumps() default twice """ - with self.assertRaises(TypeError): - orjson.dumps({}, default, default=default) + with pytest.raises(TypeError): + orjson.dumps({}, default, default=default) # type: ignore def test_option_twice(self): """ dumps() option twice """ - with self.assertRaises(TypeError): - orjson.dumps({}, None, orjson.OPT_NAIVE_UTC, option=orjson.OPT_NAIVE_UTC) + with pytest.raises(TypeError): + orjson.dumps({}, None, orjson.OPT_NAIVE_UTC, option=orjson.OPT_NAIVE_UTC) # type: ignore def test_option_mixed(self): """ @@ -152,43 +230,45 @@ class Custom: def __str__(self): return "zxc" - self.assertEqual( + assert ( orjson.dumps( [Custom(), datetime.datetime(2000, 1, 1, 2, 3, 4)], default, option=orjson.OPT_NAIVE_UTC, - ), - b'["zxc","2000-01-01T02:03:04+00:00"]', + ) + == b'["zxc","2000-01-01T02:03:04+00:00"]' ) def test_dumps_signature(self): """ dumps() valid __text_signature__ """ - self.assertEqual( - str(inspect.signature(orjson.dumps)), "(obj, /, default=None, option=None)" + assert ( + str(inspect.signature(orjson.dumps)) + == "(obj, /, default=None, option=None)" ) inspect.signature(orjson.dumps).bind("str") inspect.signature(orjson.dumps).bind("str", default=default, option=1) + inspect.signature(orjson.dumps).bind("str", default=None, option=None) def test_loads_signature(self): """ loads() valid __text_signature__ """ - self.assertEqual(str(inspect.signature(orjson.loads)), "(obj, /)") + assert str(inspect.signature(orjson.loads)), "(obj == /)" inspect.signature(orjson.loads).bind("[]") def test_dumps_module_str(self): """ orjson.dumps.__module__ is a str """ - self.assertEqual(orjson.dumps.__module__, "orjson") + assert orjson.dumps.__module__ == "orjson" def test_loads_module_str(self): """ orjson.loads.__module__ is a str """ - self.assertEqual(orjson.loads.__module__, "orjson") + assert orjson.loads.__module__ == "orjson" def test_bytes_buffer(self): """ @@ -197,9 +277,7 @@ def test_bytes_buffer(self): a = "a" * 900 b = "b" * 4096 c = "c" * 4096 * 4096 - self.assertEqual( - orjson.dumps([a, b, c]), f'["{a}","{b}","{c}"]'.encode("utf-8") - ) + assert orjson.dumps([a, b, c]) == f'["{a}","{b}","{c}"]'.encode("utf-8") def test_bytes_null_terminated(self): """ diff --git a/test/test_append_newline.py b/test/test_append_newline.py index c7f20712..ebec39f2 100644 --- a/test/test_append_newline.py +++ b/test/test_append_newline.py @@ -1,51 +1,46 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import unittest +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2020-2025) import orjson -from .util import read_fixture_obj +from .util import needs_data, read_fixture_obj -class AppendNewlineTests(unittest.TestCase): +class TestAppendNewline: def test_dumps_newline(self): """ dumps() OPT_APPEND_NEWLINE """ - self.assertEqual(orjson.dumps([], option=orjson.OPT_APPEND_NEWLINE), b"[]\n") + assert orjson.dumps([], option=orjson.OPT_APPEND_NEWLINE) == b"[]\n" + @needs_data def test_twitter_newline(self): """ loads(),dumps() twitter.json OPT_APPEND_NEWLINE """ val = read_fixture_obj("twitter.json.xz") - self.assertEqual( - orjson.loads(orjson.dumps(val, option=orjson.OPT_APPEND_NEWLINE)), val - ) + assert orjson.loads(orjson.dumps(val, option=orjson.OPT_APPEND_NEWLINE)) == val + @needs_data def test_canada(self): """ loads(), dumps() canada.json OPT_APPEND_NEWLINE """ val = read_fixture_obj("canada.json.xz") - self.assertEqual( - orjson.loads(orjson.dumps(val, option=orjson.OPT_APPEND_NEWLINE)), val - ) + assert orjson.loads(orjson.dumps(val, option=orjson.OPT_APPEND_NEWLINE)) == val + @needs_data def test_citm_catalog_newline(self): """ loads(), dumps() citm_catalog.json OPT_APPEND_NEWLINE """ val = read_fixture_obj("citm_catalog.json.xz") - self.assertEqual( - orjson.loads(orjson.dumps(val, option=orjson.OPT_APPEND_NEWLINE)), val - ) + assert orjson.loads(orjson.dumps(val, option=orjson.OPT_APPEND_NEWLINE)) == val + @needs_data def test_github_newline(self): """ loads(), dumps() github.json OPT_APPEND_NEWLINE """ val = read_fixture_obj("github.json.xz") - self.assertEqual( - orjson.loads(orjson.dumps(val, option=orjson.OPT_APPEND_NEWLINE)), val - ) + assert orjson.loads(orjson.dumps(val, option=orjson.OPT_APPEND_NEWLINE)) == val diff --git a/test/test_buffer.py b/test/test_buffer.py new file mode 100644 index 00000000..1e9239f3 --- /dev/null +++ b/test/test_buffer.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2025) + +import os + +import pytest + +import orjson + +ORJSON_RUNNER_MEMORY_GIB = os.getenv("ORJSON_RUNNER_MEMORY_GIB", "") + + +@pytest.mark.skipif( + not ORJSON_RUNNER_MEMORY_GIB, + reason="ORJSON_RUNNER_MEMORY_GIB not defined", +) +def test_memory_loads(): + buffer_factor = 12 + segment = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + size = ( + (int(ORJSON_RUNNER_MEMORY_GIB) * 1024 * 1024 * 1024) + // buffer_factor + // len(segment) + ) + doc = "".join(segment for _ in range(size)) + with pytest.raises(orjson.JSONDecodeError) as exc_info: + _ = orjson.loads(doc) + assert ( + str(exc_info.value) + == "Not enough memory to allocate buffer for parsing: line 1 column 1 (char 0)" + ) diff --git a/test/test_canonical.py b/test/test_canonical.py index 32b6854c..d07a983b 100644 --- a/test/test_canonical.py +++ b/test/test_canonical.py @@ -1,28 +1,27 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import unittest +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2019-2022) import orjson -class CanonicalTests(unittest.TestCase): +class TestCanonicalTests: def test_dumps_ctrl_escape(self): """ dumps() ctrl characters """ - self.assertEqual(orjson.dumps("text\u0003\r\n"), b'"text\\u0003\\r\\n"') + assert orjson.dumps("text\u0003\r\n") == b'"text\\u0003\\r\\n"' def test_dumps_escape_quote_backslash(self): """ dumps() quote, backslash escape """ - self.assertEqual(orjson.dumps(r'"\ test'), b'"\\"\\\\ test"') + assert orjson.dumps(r'"\ test') == b'"\\"\\\\ test"' def test_dumps_escape_line_separator(self): """ dumps() U+2028, U+2029 escape """ - self.assertEqual( - orjson.dumps({"spaces": "\u2028 \u2029"}), - b'{"spaces":"\xe2\x80\xa8 \xe2\x80\xa9"}', + assert ( + orjson.dumps({"spaces": "\u2028 \u2029"}) + == b'{"spaces":"\xe2\x80\xa8 \xe2\x80\xa9"}' ) diff --git a/test/test_circular.py b/test/test_circular.py index f24ca790..6bc1ec6c 100644 --- a/test/test_circular.py +++ b/test/test_circular.py @@ -1,34 +1,71 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2019-2023) -import unittest +import pytest import orjson -class CircularTests(unittest.TestCase): +class TestCircular: def test_circular_dict(self): """ dumps() circular reference dict """ - obj = {} + obj = {} # type: ignore obj["obj"] = obj - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(obj) + def test_circular_dict_sort_keys(self): + """ + dumps() circular reference dict OPT_SORT_KEYS + """ + obj = {} # type: ignore + obj["obj"] = obj + with pytest.raises(orjson.JSONEncodeError): + orjson.dumps(obj, option=orjson.OPT_SORT_KEYS) + + def test_circular_dict_non_str_keys(self): + """ + dumps() circular reference dict OPT_NON_STR_KEYS + """ + obj = {} # type: ignore + obj["obj"] = obj + with pytest.raises(orjson.JSONEncodeError): + orjson.dumps(obj, option=orjson.OPT_NON_STR_KEYS) + def test_circular_list(self): """ dumps() circular reference list """ - obj = [] - obj.append(obj) - with self.assertRaises(orjson.JSONEncodeError): + obj = [] # type: ignore + obj.append(obj) # type: ignore + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(obj) def test_circular_nested(self): """ dumps() circular reference nested dict, list """ - obj = {} + obj = {} # type: ignore obj["list"] = [{"obj": obj}] - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(obj) + + def test_circular_nested_sort_keys(self): + """ + dumps() circular reference nested dict, list OPT_SORT_KEYS + """ + obj = {} # type: ignore + obj["list"] = [{"obj": obj}] + with pytest.raises(orjson.JSONEncodeError): + orjson.dumps(obj, option=orjson.OPT_SORT_KEYS) + + def test_circular_nested_non_str_keys(self): + """ + dumps() circular reference nested dict, list OPT_NON_STR_KEYS + """ + obj = {} # type: ignore + obj["list"] = [{"obj": obj}] + with pytest.raises(orjson.JSONEncodeError): + orjson.dumps(obj, option=orjson.OPT_NON_STR_KEYS) diff --git a/test/test_dataclass.py b/test/test_dataclass.py index f47074b2..d8017700 100644 --- a/test/test_dataclass.py +++ b/test/test_dataclass.py @@ -1,11 +1,13 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2019-2026) import abc -import unittest import uuid from dataclasses import InitVar, asdict, dataclass, field from enum import Enum -from typing import ClassVar, Dict, Optional +from typing import ClassVar, Optional + +import pytest import orjson @@ -34,7 +36,7 @@ class Dataclass1: @dataclass class Dataclass2: - name: Optional[str] = field(default="?") + name: str | None = field(default="?") @dataclass @@ -62,7 +64,7 @@ class Datasubclass(Dataclass1): @dataclass class Slotsdataclass: - __slots__ = ("a", "b", "_c", "d") + __slots__ = ("_c", "a", "b", "d") a: str b: int _c: str @@ -81,7 +83,7 @@ class UnsortedDataclass: c: int b: int a: int - d: Optional[Dict] + d: dict | None @dataclass @@ -104,7 +106,6 @@ def key(self): @dataclass(frozen=True) class ConcreteAbc(AbstractBase): - __slots__ = ("attr",) attr: float @@ -113,25 +114,22 @@ def key(self): return "dkjf" -class DataclassTests(unittest.TestCase): +class TestDataclass: def test_dataclass(self): """ dumps() dataclass """ obj = Dataclass1("a", 1, None) - self.assertEqual( - orjson.dumps(obj), - b'{"name":"a","number":1,"sub":null}', - ) + assert orjson.dumps(obj) == b'{"name":"a","number":1,"sub":null}' def test_dataclass_recursive(self): """ dumps() dataclass recursive """ obj = Dataclass1("a", 1, Dataclass1("b", 2, None)) - self.assertEqual( - orjson.dumps(obj), - b'{"name":"a","number":1,"sub":{"name":"b","number":2,"sub":null}}', + assert ( + orjson.dumps(obj) + == b'{"name":"a","number":1,"sub":{"name":"b","number":2,"sub":null}}' ) def test_dataclass_circular(self): @@ -141,42 +139,36 @@ def test_dataclass_circular(self): obj1 = Dataclass1("a", 1, None) obj2 = Dataclass1("b", 2, obj1) obj1.sub = obj2 - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(obj1) def test_dataclass_empty(self): """ dumps() no attributes """ - self.assertEqual( - orjson.dumps(EmptyDataclass()), - b"{}", - ) + assert orjson.dumps(EmptyDataclass()) == b"{}" def test_dataclass_empty_slots(self): """ dumps() no attributes slots """ - self.assertEqual( - orjson.dumps(EmptyDataclassSlots()), - b"{}", - ) + assert orjson.dumps(EmptyDataclassSlots()) == b"{}" def test_dataclass_default_arg(self): """ dumps() dataclass default arg """ obj = Dataclass2() - self.assertEqual(orjson.dumps(obj), b'{"name":"?"}') + assert orjson.dumps(obj) == b'{"name":"?"}' def test_dataclass_types(self): """ dumps() dataclass types """ obj = Dataclass3("a", 1, {"a": "b"}, True, 1.1, [1, 2], (3, 4)) - self.assertEqual( - orjson.dumps(obj), - b'{"a":"a","b":1,"c":{"a":"b"},"d":true,"e":1.1,"f":[1,2],"g":[3,4]}', + assert ( + orjson.dumps(obj) + == b'{"a":"a","b":1,"c":{"a":"b"},"d":true,"e":1.1,"f":[1,2],"g":[3,4]}' ) def test_dataclass_metadata(self): @@ -184,29 +176,23 @@ def test_dataclass_metadata(self): dumps() dataclass metadata """ obj = Dataclass4("a", 1, 2.1) - self.assertEqual( - orjson.dumps(obj), - b'{"a":"a","b":1,"c":2.1}', - ) + assert orjson.dumps(obj) == b'{"a":"a","b":1,"c":2.1}' def test_dataclass_classvar(self): """ dumps() dataclass class variable """ obj = Dataclass4("a", 1) - self.assertEqual( - orjson.dumps(obj), - b'{"a":"a","b":1,"c":1.1}', - ) + assert orjson.dumps(obj) == b'{"a":"a","b":1,"c":1.1}' def test_dataclass_subclass(self): """ dumps() dataclass subclass """ obj = Datasubclass("a", 1, None, False) - self.assertEqual( - orjson.dumps(obj), - b'{"name":"a","number":1,"sub":null,"additional":false}', + assert ( + orjson.dumps(obj) + == b'{"name":"a","number":1,"sub":null,"additional":false}' ) def test_dataclass_slots(self): @@ -215,7 +201,7 @@ def test_dataclass_slots(self): """ obj = Slotsdataclass("a", 1, "c", "d") assert "__dict__" not in dir(obj) - self.assertEqual(orjson.dumps(obj), b'{"a":"a","b":1}') + assert orjson.dumps(obj) == b'{"a":"a","b":1}' def test_dataclass_default(self): """ @@ -229,11 +215,12 @@ def default(__obj): return __obj.value obj = Defaultdataclass( - uuid.UUID("808989c0-00d5-48a8-b5c4-c804bf9032f2"), AnEnum.ONE + uuid.UUID("808989c0-00d5-48a8-b5c4-c804bf9032f2"), + AnEnum.ONE, ) - self.assertEqual( - orjson.dumps(obj, default=default), - b'{"a":"808989c0-00d5-48a8-b5c4-c804bf9032f2","b":1}', + assert ( + orjson.dumps(obj, default=default) + == b'{"a":"808989c0-00d5-48a8-b5c4-c804bf9032f2","b":1}' ) def test_dataclass_sort(self): @@ -241,9 +228,9 @@ def test_dataclass_sort(self): OPT_SORT_KEYS has no effect on dataclasses """ obj = UnsortedDataclass(1, 2, 3, None) - self.assertEqual( - orjson.dumps(obj, option=orjson.OPT_SORT_KEYS), - b'{"c":1,"b":2,"a":3,"d":null}', + assert ( + orjson.dumps(obj, option=orjson.OPT_SORT_KEYS) + == b'{"c":1,"b":2,"a":3,"d":null}' ) def test_dataclass_sort_sub(self): @@ -251,9 +238,9 @@ def test_dataclass_sort_sub(self): dataclass fast path does not prevent OPT_SORT_KEYS from cascading """ obj = UnsortedDataclass(1, 2, 3, {"f": 2, "e": 1}) - self.assertEqual( - orjson.dumps(obj, option=orjson.OPT_SORT_KEYS), - b'{"c":1,"b":2,"a":3,"d":{"e":1,"f":2}}', + assert ( + orjson.dumps(obj, option=orjson.OPT_SORT_KEYS) + == b'{"c":1,"b":2,"a":3,"d":{"e":1,"f":2}}' ) def test_dataclass_under(self): @@ -261,33 +248,31 @@ def test_dataclass_under(self): dumps() does not include under attributes, InitVar, or ClassVar """ obj = InitDataclass("zxc", "vbn") - self.assertEqual( - orjson.dumps(obj), - b'{"ab":"zxc vbn"}', - ) + assert orjson.dumps(obj) == b'{"ab":"zxc vbn"}' def test_dataclass_option(self): """ dumps() accepts deprecated OPT_SERIALIZE_DATACLASS """ obj = Dataclass1("a", 1, None) - self.assertEqual( - orjson.dumps(obj, option=orjson.OPT_SERIALIZE_DATACLASS), - b'{"name":"a","number":1,"sub":null}', + assert ( + orjson.dumps(obj, option=orjson.OPT_SERIALIZE_DATACLASS) + == b'{"name":"a","number":1,"sub":null}' ) -class DataclassPassthroughTests(unittest.TestCase): +class TestDataclassPassthrough: def test_dataclass_passthrough_raise(self): """ dumps() dataclass passes to default with OPT_PASSTHROUGH_DATACLASS """ obj = Dataclass1("a", 1, None) - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(obj, option=orjson.OPT_PASSTHROUGH_DATACLASS) - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps( - InitDataclass("zxc", "vbn"), option=orjson.OPT_PASSTHROUGH_DATACLASS + InitDataclass("zxc", "vbn"), + option=orjson.OPT_PASSTHROUGH_DATACLASS, ) def test_dataclass_passthrough_default(self): @@ -295,9 +280,9 @@ def test_dataclass_passthrough_default(self): dumps() dataclass passes to default with OPT_PASSTHROUGH_DATACLASS """ obj = Dataclass1("a", 1, None) - self.assertEqual( - orjson.dumps(obj, option=orjson.OPT_PASSTHROUGH_DATACLASS, default=asdict), - b'{"name":"a","number":1,"sub":null}', + assert ( + orjson.dumps(obj, option=orjson.OPT_PASSTHROUGH_DATACLASS, default=asdict) + == b'{"name":"a","number":1,"sub":null}' ) def default(obj): @@ -305,13 +290,13 @@ def default(obj): return {"name": obj.name, "number": obj.number} raise TypeError - self.assertEqual( - orjson.dumps(obj, option=orjson.OPT_PASSTHROUGH_DATACLASS, default=default), - b'{"name":"a","number":1}', + assert ( + orjson.dumps(obj, option=orjson.OPT_PASSTHROUGH_DATACLASS, default=default) + == b'{"name":"a","number":1}' ) -class AbstractDataclassTests(unittest.TestCase): +class TestAbstractDataclass: def test_dataclass_abc(self): obj = ConcreteAbc(1.0) - self.assertEqual(orjson.dumps(obj), b'{"attr":1.0}') + assert orjson.dumps(obj) == b'{"attr":1.0}' diff --git a/test/test_datetime.py b/test/test_datetime.py index ebd7bd85..d1e8211f 100644 --- a/test/test_datetime.py +++ b/test/test_datetime.py @@ -1,17 +1,17 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2019-2026) import datetime -import sys -import unittest import pytest -from dateutil import tz import orjson try: import zoneinfo -except ImportError: + + _ = zoneinfo.ZoneInfo("Europe/Amsterdam") +except Exception: # ImportError,ZoneInfoNotFoundError zoneinfo = None # type: ignore try: @@ -24,109 +24,151 @@ except ImportError: pendulum = None # type: ignore -if sys.version_info >= (3, 9): - import zoneinfo +try: + from dateutil import tz +except ImportError: + tz = None # type: ignore -class DatetimeTests(unittest.TestCase): +AMSTERDAM_1937_DATETIMES = ( + b'["1937-01-01T12:00:27.000087+00:20"]', # tzinfo<2022b and an example in RFC 3339 + b'["1937-01-01T12:00:27.000087+00:00"]', # tzinfo>=2022b +) + +AMSTERDAM_1937_DATETIMES_WITH_Z = ( + b'["1937-01-01T12:00:27.000087+00:20"]', + b'["1937-01-01T12:00:27.000087Z"]', +) + + +class TestDatetime: def test_datetime_naive(self): """ datetime.datetime naive prints without offset """ - self.assertEqual( - orjson.dumps([datetime.datetime(2000, 1, 1, 2, 3, 4, 123)]), - b'["2000-01-01T02:03:04.000123"]', + assert ( + orjson.dumps([datetime.datetime(2000, 1, 1, 2, 3, 4, 123)]) + == b'["2000-01-01T02:03:04.000123"]' ) def test_datetime_naive_utc(self): """ datetime.datetime naive with opt assumes UTC """ - self.assertEqual( + assert ( orjson.dumps( [datetime.datetime(2000, 1, 1, 2, 3, 4, 123)], option=orjson.OPT_NAIVE_UTC, - ), - b'["2000-01-01T02:03:04.000123+00:00"]', + ) + == b'["2000-01-01T02:03:04.000123+00:00"]' ) def test_datetime_min(self): """ datetime.datetime min range """ - self.assertEqual( + assert ( orjson.dumps( [datetime.datetime(datetime.MINYEAR, 1, 1, 0, 0, 0, 0)], option=orjson.OPT_NAIVE_UTC, - ), - b'["0001-01-01T00:00:00+00:00"]', + ) + == b'["0001-01-01T00:00:00+00:00"]' ) + def test_datetime_min_invalid(self): + """ + datetime.datetime min range invalid + """ + with pytest.raises(ValueError): + datetime.datetime(-1, 1, 1, 0, 0, 0, 0) + + def test_datetime_max_invalid(self): + """ + datetime.datetime max range invalid + """ + with pytest.raises(ValueError): + datetime.datetime(10000, 1, 1, 0, 0, 0, 0) + def test_datetime_max(self): """ datetime.datetime max range """ - self.assertEqual( + assert ( orjson.dumps( [datetime.datetime(datetime.MAXYEAR, 12, 31, 23, 59, 50, 999999)], option=orjson.OPT_NAIVE_UTC, - ), - b'["9999-12-31T23:59:50.999999+00:00"]', + ) + == b'["9999-12-31T23:59:50.999999+00:00"]' ) def test_datetime_three_digits(self): """ datetime.datetime three digit year """ - self.assertEqual( + assert ( orjson.dumps( [datetime.datetime(312, 1, 1)], option=orjson.OPT_NAIVE_UTC, - ), - b'["0312-01-01T00:00:00+00:00"]', + ) + == b'["0312-01-01T00:00:00+00:00"]' ) def test_datetime_two_digits(self): """ datetime.datetime two digit year """ - self.assertEqual( + assert ( orjson.dumps( [datetime.datetime(46, 1, 1)], option=orjson.OPT_NAIVE_UTC, - ), - b'["0046-01-01T00:00:00+00:00"]', + ) + == b'["0046-01-01T00:00:00+00:00"]' ) + @pytest.mark.skipif(tz is None, reason="dateutil optional") def test_datetime_tz_assume(self): """ datetime.datetime tz with assume UTC uses tz """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( - 2018, 1, 1, 2, 3, 4, 0, tzinfo=tz.gettz("Asia/Shanghai") - ) + 2018, + 1, + 1, + 2, + 3, + 4, + 0, + tzinfo=tz.gettz("Asia/Shanghai"), + ), ], option=orjson.OPT_NAIVE_UTC, - ), - b'["2018-01-01T02:03:04+08:00"]', + ) + == b'["2018-01-01T02:03:04+08:00"]' ) def test_datetime_timezone_utc(self): """ datetime.datetime.utc """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( - 2018, 6, 1, 2, 3, 4, 0, tzinfo=datetime.timezone.utc - ) - ] - ), - b'["2018-06-01T02:03:04+00:00"]', + 2018, + 6, + 1, + 2, + 3, + 4, + 0, + tzinfo=datetime.timezone.utc, + ), + ], + ) + == b'["2018-06-01T02:03:04+00:00"]' ) @pytest.mark.skipif(pytz is None, reason="pytz optional") @@ -134,36 +176,37 @@ def test_datetime_pytz_utc(self): """ pytz.UTC """ - self.assertEqual( - orjson.dumps([datetime.datetime(2018, 6, 1, 2, 3, 4, 0, tzinfo=pytz.UTC)]), - b'["2018-06-01T02:03:04+00:00"]', + assert ( + orjson.dumps([datetime.datetime(2018, 6, 1, 2, 3, 4, 0, tzinfo=pytz.UTC)]) + == b'["2018-06-01T02:03:04+00:00"]' ) - @unittest.skipIf( - sys.version_info < (3, 9) or sys.platform.startswith("win"), - "zoneinfo not available", - ) + @pytest.mark.skipif(zoneinfo is None, reason="zoneinfo not available") def test_datetime_zoneinfo_utc(self): """ zoneinfo.ZoneInfo("UTC") """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( - 2018, 6, 1, 2, 3, 4, 0, tzinfo=zoneinfo.ZoneInfo("UTC") - ) - ] - ), - b'["2018-06-01T02:03:04+00:00"]', + 2018, + 6, + 1, + 2, + 3, + 4, + 0, + tzinfo=zoneinfo.ZoneInfo("UTC"), + ), + ], + ) + == b'["2018-06-01T02:03:04+00:00"]' ) - @unittest.skipIf( - sys.version_info < (3, 9) or sys.platform.startswith("win"), - "zoneinfo not available", - ) + @pytest.mark.skipif(zoneinfo is None, reason="zoneinfo not available") def test_datetime_zoneinfo_positive(self): - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( @@ -175,18 +218,15 @@ def test_datetime_zoneinfo_positive(self): 4, 0, tzinfo=zoneinfo.ZoneInfo("Asia/Shanghai"), - ) - ] - ), - b'["2018-01-01T02:03:04+08:00"]', + ), + ], + ) + == b'["2018-01-01T02:03:04+08:00"]' ) - @unittest.skipIf( - sys.version_info < (3, 9) or sys.platform.startswith("win"), - "zoneinfo not available", - ) + @pytest.mark.skipif(zoneinfo is None, reason="zoneinfo not available") def test_datetime_zoneinfo_negative(self): - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( @@ -198,37 +238,45 @@ def test_datetime_zoneinfo_negative(self): 4, 0, tzinfo=zoneinfo.ZoneInfo("America/New_York"), - ) - ] - ), - b'["2018-06-01T02:03:04-04:00"]', + ), + ], + ) + == b'["2018-06-01T02:03:04-04:00"]' ) - @pytest.mark.skipif(pendulum is None, reason="pendulum install broken on win") + @pytest.mark.skipif(pendulum is None, reason="pendulum not installed") def test_datetime_pendulum_utc(self): """ datetime.datetime UTC """ - self.assertEqual( + assert ( orjson.dumps( - [datetime.datetime(2018, 6, 1, 2, 3, 4, 0, tzinfo=pendulum.UTC)] - ), - b'["2018-06-01T02:03:04+00:00"]', + [datetime.datetime(2018, 6, 1, 2, 3, 4, 0, tzinfo=pendulum.UTC)], + ) + == b'["2018-06-01T02:03:04+00:00"]' ) + @pytest.mark.skipif(tz is None, reason="dateutil optional") def test_datetime_arrow_positive(self): """ datetime.datetime positive UTC """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( - 2018, 1, 1, 2, 3, 4, 0, tzinfo=tz.gettz("Asia/Shanghai") - ) - ] - ), - b'["2018-01-01T02:03:04+08:00"]', + 2018, + 1, + 1, + 2, + 3, + 4, + 0, + tzinfo=tz.gettz("Asia/Shanghai"), + ), + ], + ) + == b'["2018-01-01T02:03:04+08:00"]' ) @pytest.mark.skipif(pytz is None, reason="pytz optional") @@ -236,23 +284,30 @@ def test_datetime_pytz_positive(self): """ datetime.datetime positive UTC """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( - 2018, 1, 1, 2, 3, 4, 0, tzinfo=pytz.timezone("Asia/Shanghai") - ) - ] - ), - b'["2018-01-01T02:03:04+08:00"]', + 2018, + 1, + 1, + 2, + 3, + 4, + 0, + tzinfo=pytz.timezone("Asia/Shanghai"), + ), + ], + ) + == b'["2018-01-01T02:03:04+08:00"]' ) - @pytest.mark.skipif(pendulum is None, reason="pendulum install broken on win") + @pytest.mark.skipif(pendulum is None, reason="pendulum not installed") def test_datetime_pendulum_positive(self): """ datetime.datetime positive UTC """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( @@ -263,11 +318,11 @@ def test_datetime_pendulum_positive(self): 3, 4, 0, - tzinfo=pendulum.timezone("Asia/Shanghai"), - ) - ] - ), - b'["2018-01-01T02:03:04+08:00"]', + tzinfo=pendulum.timezone("Asia/Shanghai"), # type: ignore + ), + ], + ) + == b'["2018-01-01T02:03:04+08:00"]' ) @pytest.mark.skipif(pytz is None, reason="pytz optional") @@ -275,23 +330,30 @@ def test_datetime_pytz_negative_dst(self): """ datetime.datetime negative UTC DST """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( - 2018, 6, 1, 2, 3, 4, 0, tzinfo=pytz.timezone("America/New_York") - ) - ] - ), - b'["2018-06-01T02:03:04-04:00"]', + 2018, + 6, + 1, + 2, + 3, + 4, + 0, + tzinfo=pytz.timezone("America/New_York"), + ), + ], + ) + == b'["2018-06-01T02:03:04-04:00"]' ) - @pytest.mark.skipif(pendulum is None, reason="pendulum install broken on win") + @pytest.mark.skipif(pendulum is None, reason="pendulum not installed") def test_datetime_pendulum_negative_dst(self): """ datetime.datetime negative UTC DST """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( @@ -302,22 +364,19 @@ def test_datetime_pendulum_negative_dst(self): 3, 4, 0, - tzinfo=pendulum.timezone("America/New_York"), - ) - ] - ), - b'["2018-06-01T02:03:04-04:00"]', + tzinfo=pendulum.timezone("America/New_York"), # type: ignore + ), + ], + ) + == b'["2018-06-01T02:03:04-04:00"]' ) - @unittest.skipIf( - sys.version_info < (3, 9) or sys.platform.startswith("win"), - "zoneinfo not available", - ) + @pytest.mark.skipif(zoneinfo is None, reason="zoneinfo not available") def test_datetime_zoneinfo_negative_non_dst(self): """ datetime.datetime negative UTC non-DST """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( @@ -329,10 +388,10 @@ def test_datetime_zoneinfo_negative_non_dst(self): 4, 0, tzinfo=zoneinfo.ZoneInfo("America/New_York"), - ) - ] - ), - b'["2018-12-01T02:03:04-05:00"]', + ), + ], + ) + == b'["2018-12-01T02:03:04-05:00"]' ) @pytest.mark.skipif(pytz is None, reason="pytz optional") @@ -340,7 +399,7 @@ def test_datetime_pytz_negative_non_dst(self): """ datetime.datetime negative UTC non-DST """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( @@ -352,18 +411,18 @@ def test_datetime_pytz_negative_non_dst(self): 4, 0, tzinfo=pytz.timezone("America/New_York"), - ) - ] - ), - b'["2018-12-01T02:03:04-05:00"]', + ), + ], + ) + == b'["2018-12-01T02:03:04-05:00"]' ) - @pytest.mark.skipif(pendulum is None, reason="pendulum install broken on win") + @pytest.mark.skipif(pendulum is None, reason="pendulum not installed") def test_datetime_pendulum_negative_non_dst(self): """ datetime.datetime negative UTC non-DST """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( @@ -374,22 +433,19 @@ def test_datetime_pendulum_negative_non_dst(self): 3, 4, 0, - tzinfo=pendulum.timezone("America/New_York"), - ) - ] - ), - b'["2018-12-01T02:03:04-05:00"]', + tzinfo=pendulum.timezone("America/New_York"), # type: ignore + ), + ], + ) + == b'["2018-12-01T02:03:04-05:00"]' ) - @unittest.skipIf( - sys.version_info < (3, 9) or sys.platform.startswith("win"), - "zoneinfo not available", - ) + @pytest.mark.skipif(zoneinfo is None, reason="zoneinfo not available") def test_datetime_zoneinfo_partial_hour(self): """ datetime.datetime UTC offset partial hour """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( @@ -401,10 +457,10 @@ def test_datetime_zoneinfo_partial_hour(self): 4, 0, tzinfo=zoneinfo.ZoneInfo("Australia/Adelaide"), - ) - ] - ), - b'["2018-12-01T02:03:04+10:30"]', + ), + ], + ) + == b'["2018-12-01T02:03:04+10:30"]' ) @pytest.mark.skipif(pytz is None, reason="pytz optional") @@ -412,7 +468,7 @@ def test_datetime_pytz_partial_hour(self): """ datetime.datetime UTC offset partial hour """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( @@ -424,18 +480,18 @@ def test_datetime_pytz_partial_hour(self): 4, 0, tzinfo=pytz.timezone("Australia/Adelaide"), - ) - ] - ), - b'["2018-12-01T02:03:04+10:30"]', + ), + ], + ) + == b'["2018-12-01T02:03:04+10:30"]' ) - @pytest.mark.skipif(pendulum is None, reason="pendulum install broken on win") + @pytest.mark.skipif(pendulum is None, reason="pendulum not installed") def test_datetime_pendulum_partial_hour(self): """ datetime.datetime UTC offset partial hour """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( @@ -446,21 +502,21 @@ def test_datetime_pendulum_partial_hour(self): 3, 4, 0, - tzinfo=pendulum.timezone("Australia/Adelaide"), - ) - ] - ), - b'["2018-12-01T02:03:04+10:30"]', + tzinfo=pendulum.timezone("Australia/Adelaide"), # type: ignore + ), + ], + ) + == b'["2018-12-01T02:03:04+10:30"]' ) - @pytest.mark.skipif(pendulum is None, reason="pendulum install broken on win") + @pytest.mark.skipif(pendulum is None, reason="pendulum not installed") def test_datetime_partial_second_pendulum_supported(self): """ datetime.datetime UTC offset round seconds https://tools.ietf.org/html/rfc3339#section-5.8 """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( @@ -471,24 +527,21 @@ def test_datetime_partial_second_pendulum_supported(self): 0, 27, 87, - tzinfo=pendulum.timezone("Europe/Amsterdam"), - ) - ] - ), - b'["1937-01-01T12:00:27.000087+00:20"]', + tzinfo=pendulum.timezone("Europe/Amsterdam"), # type: ignore + ), + ], + ) + in AMSTERDAM_1937_DATETIMES ) - @unittest.skipIf( - sys.version_info < (3, 9) or sys.platform.startswith("win"), - "zoneinfo not available", - ) + @pytest.mark.skipif(zoneinfo is None, reason="zoneinfo not available") def test_datetime_partial_second_zoneinfo(self): """ datetime.datetime UTC offset round seconds https://tools.ietf.org/html/rfc3339#section-5.8 """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( @@ -500,10 +553,10 @@ def test_datetime_partial_second_zoneinfo(self): 27, 87, tzinfo=zoneinfo.ZoneInfo("Europe/Amsterdam"), - ) - ] - ), - b'["1937-01-01T12:00:27.000087+00:20"]', + ), + ], + ) + in AMSTERDAM_1937_DATETIMES ) @pytest.mark.skipif(pytz is None, reason="pytz optional") @@ -513,7 +566,7 @@ def test_datetime_partial_second_pytz(self): https://tools.ietf.org/html/rfc3339#section-5.8 """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( @@ -525,147 +578,172 @@ def test_datetime_partial_second_pytz(self): 27, 87, tzinfo=pytz.timezone("Europe/Amsterdam"), - ) - ] - ), - b'["1937-01-01T12:00:27.000087+00:20"]', + ), + ], + ) + in AMSTERDAM_1937_DATETIMES ) + @pytest.mark.skipif(tz is None, reason="dateutil optional") def test_datetime_partial_second_dateutil(self): """ datetime.datetime UTC offset round seconds https://tools.ietf.org/html/rfc3339#section-5.8 """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( - 1937, 1, 1, 12, 0, 27, 87, tzinfo=tz.gettz("Europe/Amsterdam") - ) - ] - ), - b'["1937-01-01T12:00:27.000087+00:20"]', + 1937, + 1, + 1, + 12, + 0, + 27, + 87, + tzinfo=tz.gettz("Europe/Amsterdam"), + ), + ], + ) + in AMSTERDAM_1937_DATETIMES ) def test_datetime_microsecond_max(self): """ datetime.datetime microsecond max """ - self.assertEqual( - orjson.dumps(datetime.datetime(2000, 1, 1, 0, 0, 0, 999999)), - b'"2000-01-01T00:00:00.999999"', + assert ( + orjson.dumps(datetime.datetime(2000, 1, 1, 0, 0, 0, 999999)) + == b'"2000-01-01T00:00:00.999999"' ) def test_datetime_microsecond_min(self): """ datetime.datetime microsecond min """ - self.assertEqual( - orjson.dumps(datetime.datetime(2000, 1, 1, 0, 0, 0, 1)), - b'"2000-01-01T00:00:00.000001"', + assert ( + orjson.dumps(datetime.datetime(2000, 1, 1, 0, 0, 0, 1)) + == b'"2000-01-01T00:00:00.000001"' ) def test_datetime_omit_microseconds(self): """ datetime.datetime OPT_OMIT_MICROSECONDS """ - self.assertEqual( + assert ( orjson.dumps( [datetime.datetime(2000, 1, 1, 2, 3, 4, 123)], option=orjson.OPT_OMIT_MICROSECONDS, - ), - b'["2000-01-01T02:03:04"]', + ) + == b'["2000-01-01T02:03:04"]' ) def test_datetime_omit_microseconds_naive(self): """ datetime.datetime naive OPT_OMIT_MICROSECONDS """ - self.assertEqual( + assert ( orjson.dumps( [datetime.datetime(2000, 1, 1, 2, 3, 4, 123)], option=orjson.OPT_NAIVE_UTC | orjson.OPT_OMIT_MICROSECONDS, - ), - b'["2000-01-01T02:03:04+00:00"]', + ) + == b'["2000-01-01T02:03:04+00:00"]' ) def test_time_omit_microseconds(self): """ datetime.time OPT_OMIT_MICROSECONDS """ - self.assertEqual( + assert ( orjson.dumps( - [datetime.time(2, 3, 4, 123)], option=orjson.OPT_OMIT_MICROSECONDS - ), - b'["02:03:04"]', + [datetime.time(2, 3, 4, 123)], + option=orjson.OPT_OMIT_MICROSECONDS, + ) + == b'["02:03:04"]' ) def test_datetime_utc_z_naive_omit(self): """ datetime.datetime naive OPT_UTC_Z """ - self.assertEqual( + assert ( orjson.dumps( [datetime.datetime(2000, 1, 1, 2, 3, 4, 123)], option=orjson.OPT_NAIVE_UTC | orjson.OPT_UTC_Z | orjson.OPT_OMIT_MICROSECONDS, - ), - b'["2000-01-01T02:03:04Z"]', + ) + == b'["2000-01-01T02:03:04Z"]' ) def test_datetime_utc_z_naive(self): """ datetime.datetime naive OPT_UTC_Z """ - self.assertEqual( + assert ( orjson.dumps( [datetime.datetime(2000, 1, 1, 2, 3, 4, 123)], option=orjson.OPT_NAIVE_UTC | orjson.OPT_UTC_Z, - ), - b'["2000-01-01T02:03:04.000123Z"]', + ) + == b'["2000-01-01T02:03:04.000123Z"]' ) def test_datetime_utc_z_without_tz(self): """ datetime.datetime naive OPT_UTC_Z """ - self.assertEqual( + assert ( orjson.dumps( - [datetime.datetime(2000, 1, 1, 2, 3, 4, 123)], option=orjson.OPT_UTC_Z - ), - b'["2000-01-01T02:03:04.000123"]', + [datetime.datetime(2000, 1, 1, 2, 3, 4, 123)], + option=orjson.OPT_UTC_Z, + ) + == b'["2000-01-01T02:03:04.000123"]' ) + @pytest.mark.skipif(tz is None, reason="dateutil optional") def test_datetime_utc_z_with_tz(self): """ datetime.datetime naive OPT_UTC_Z """ - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( - 2000, 1, 1, 0, 0, 0, 1, tzinfo=datetime.timezone.utc - ) + 2000, + 1, + 1, + 0, + 0, + 0, + 1, + tzinfo=datetime.timezone.utc, + ), ], option=orjson.OPT_UTC_Z, - ), - b'["2000-01-01T00:00:00.000001Z"]', + ) + == b'["2000-01-01T00:00:00.000001Z"]' ) - self.assertEqual( + assert ( orjson.dumps( [ datetime.datetime( - 1937, 1, 1, 12, 0, 27, 87, tzinfo=tz.gettz("Europe/Amsterdam") - ) + 1937, + 1, + 1, + 12, + 0, + 27, + 87, + tzinfo=tz.gettz("Europe/Amsterdam"), + ), ], option=orjson.OPT_UTC_Z, - ), - b'["1937-01-01T12:00:27.000087+00:20"]', + ) + in AMSTERDAM_1937_DATETIMES_WITH_Z ) - @pytest.mark.skipif(pendulum is None, reason="pendulum install broken on win") + @pytest.mark.skipif(pendulum is None, reason="pendulum not installed") def test_datetime_roundtrip(self): """ datetime.datetime parsed by pendulum @@ -674,117 +752,125 @@ def test_datetime_roundtrip(self): serialized = orjson.dumps(obj).decode("utf-8").replace('"', "") parsed = pendulum.parse(serialized) for attr in ("year", "month", "day", "hour", "minute", "second", "microsecond"): - self.assertEqual(getattr(obj, attr), getattr(parsed, attr)) + assert getattr(obj, attr) == getattr(parsed, attr) -class DateTests(unittest.TestCase): +class TestDate: def test_date(self): """ datetime.date """ - self.assertEqual(orjson.dumps([datetime.date(2000, 1, 13)]), b'["2000-01-13"]') + assert orjson.dumps([datetime.date(2000, 1, 13)]) == b'["2000-01-13"]' def test_date_min(self): """ datetime.date MINYEAR """ - self.assertEqual( - orjson.dumps([datetime.date(datetime.MINYEAR, 1, 1)]), b'["0001-01-01"]' + assert ( + orjson.dumps([datetime.date(datetime.MINYEAR, 1, 1)]) == b'["0001-01-01"]' ) def test_date_max(self): """ datetime.date MAXYEAR """ - self.assertEqual( - orjson.dumps([datetime.date(datetime.MAXYEAR, 12, 31)]), b'["9999-12-31"]' + assert ( + orjson.dumps([datetime.date(datetime.MAXYEAR, 12, 31)]) == b'["9999-12-31"]' ) def test_date_three_digits(self): """ datetime.date three digit year """ - self.assertEqual( + assert ( orjson.dumps( [datetime.date(312, 1, 1)], - ), - b'["0312-01-01"]', + ) + == b'["0312-01-01"]' ) def test_date_two_digits(self): """ datetime.date two digit year """ - self.assertEqual( + assert ( orjson.dumps( [datetime.date(46, 1, 1)], - ), - b'["0046-01-01"]', + ) + == b'["0046-01-01"]' ) -class TimeTests(unittest.TestCase): +class TestTime: def test_time(self): """ datetime.time """ - self.assertEqual( - orjson.dumps([datetime.time(12, 15, 59, 111)]), b'["12:15:59.000111"]' - ) - self.assertEqual(orjson.dumps([datetime.time(12, 15, 59)]), b'["12:15:59"]') + assert orjson.dumps([datetime.time(12, 15, 59, 111)]) == b'["12:15:59.000111"]' + assert orjson.dumps([datetime.time(12, 15, 59)]) == b'["12:15:59"]' + @pytest.mark.skipif(zoneinfo is None, reason="zoneinfo not available") def test_time_tz(self): """ datetime.time with tzinfo error """ - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps( - [datetime.time(12, 15, 59, 111, tzinfo=tz.gettz("Asia/Shanghai"))] + [ + datetime.time( + 12, + 15, + 59, + 111, + tzinfo=zoneinfo.ZoneInfo("Asia/Shanghai"), + ), + ], ) def test_time_microsecond_max(self): """ datetime.time microsecond max """ - self.assertEqual( - orjson.dumps(datetime.time(0, 0, 0, 999999)), b'"00:00:00.999999"' - ) + assert orjson.dumps(datetime.time(0, 0, 0, 999999)) == b'"00:00:00.999999"' def test_time_microsecond_min(self): """ datetime.time microsecond min """ - self.assertEqual(orjson.dumps(datetime.time(0, 0, 0, 1)), b'"00:00:00.000001"') + assert orjson.dumps(datetime.time(0, 0, 0, 1)) == b'"00:00:00.000001"' -class DateclassPassthroughTests(unittest.TestCase): +class TestDateclassPassthrough: def test_passthrough_datetime(self): - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps( - datetime.datetime(1970, 1, 1), option=orjson.OPT_PASSTHROUGH_DATETIME + datetime.datetime(1970, 1, 1), + option=orjson.OPT_PASSTHROUGH_DATETIME, ) def test_passthrough_date(self): - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps( - datetime.date(1970, 1, 1), option=orjson.OPT_PASSTHROUGH_DATETIME + datetime.date(1970, 1, 1), + option=orjson.OPT_PASSTHROUGH_DATETIME, ) def test_passthrough_time(self): - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps( - datetime.time(12, 0, 0), option=orjson.OPT_PASSTHROUGH_DATETIME + datetime.time(12, 0, 0), + option=orjson.OPT_PASSTHROUGH_DATETIME, ) def test_passthrough_datetime_default(self): def default(obj): return obj.strftime("%a, %d %b %Y %H:%M:%S GMT") - self.assertEqual( + assert ( orjson.dumps( datetime.datetime(1970, 1, 1), option=orjson.OPT_PASSTHROUGH_DATETIME, default=default, - ), - b'"Thu, 01 Jan 1970 00:00:00 GMT"', + ) + == b'"Thu, 01 Jan 1970 00:00:00 GMT"' ) diff --git a/test/test_default.py b/test/test_default.py index ec877ecd..4e93c785 100644 --- a/test/test_default.py +++ b/test/test_default.py @@ -1,10 +1,16 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2019-2026), Rami Chowdhury (2020), Marc Mueller (2023), Jack Amadeo (2023) -import unittest +import datetime +import sys import uuid +import pytest + import orjson +from .util import SUPPORTS_GETREFCOUNT, numpy + class Custom: def __init__(self): @@ -30,22 +36,22 @@ def default_raises(obj): raise TypeError -class TypeTests(unittest.TestCase): +class TestType: def test_default_not_callable(self): """ dumps() default not callable """ - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(Custom(), default=NotImplementedError) ran = False try: orjson.dumps(Custom(), default=NotImplementedError) except Exception as err: - self.assertIsInstance(err, orjson.JSONEncodeError) - self.assertEqual(str(err), "default serializer exceeds recursion limit") + assert isinstance(err, orjson.JSONEncodeError) + assert str(err) == "default serializer exceeds recursion limit" ran = True - self.assertTrue(ran) + assert ran def test_default_func(self): """ @@ -56,15 +62,13 @@ def test_default_func(self): def default(obj): return str(obj) - self.assertEqual( - orjson.dumps(ref, default=default), b'"%s"' % str(ref).encode("utf-8") - ) + assert orjson.dumps(ref, default=default) == b'"%s"' % str(ref).encode("utf-8") def test_default_func_none(self): """ dumps() default function None ok """ - self.assertEqual(orjson.dumps(Custom(), default=lambda x: None), b"null") + assert orjson.dumps(Custom(), default=lambda x: None) == b"null" def test_default_func_empty(self): """ @@ -76,8 +80,8 @@ def default(obj): if isinstance(obj, set): return list(obj) - self.assertEqual(orjson.dumps(ref, default=default), b"null") - self.assertEqual(orjson.dumps({ref}, default=default), b"[null]") + assert orjson.dumps(ref, default=default) == b"null" + assert orjson.dumps({ref}, default=default) == b"[null]" def test_default_func_exc(self): """ @@ -87,17 +91,17 @@ def test_default_func_exc(self): def default(obj): raise NotImplementedError - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(Custom(), default=default) ran = False try: orjson.dumps(Custom(), default=default) except Exception as err: - self.assertIsInstance(err, orjson.JSONEncodeError) - self.assertEqual(str(err), "Type is not JSON serializable: Custom") + assert isinstance(err, orjson.JSONEncodeError) + assert str(err) == "Type is not JSON serializable: Custom" ran = True - self.assertTrue(ran) + assert ran def test_default_exception_type(self): """ @@ -105,7 +109,7 @@ def test_default_exception_type(self): """ ref = Custom() - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(ref, default=default_raises) def test_default_vectorcall_str(self): @@ -118,8 +122,9 @@ class SubStr(str): obj = SubStr("saasa") ref = b'"%s"' % str(obj).encode("utf-8") - self.assertEqual( - orjson.dumps(obj, option=orjson.OPT_PASSTHROUGH_SUBCLASS, default=str), ref + assert ( + orjson.dumps(obj, option=orjson.OPT_PASSTHROUGH_SUBCLASS, default=str) + == ref ) def test_default_vectorcall_list(self): @@ -128,7 +133,7 @@ def test_default_vectorcall_list(self): """ obj = {1, 2} ref = b"[1,2]" - self.assertEqual(orjson.dumps(obj, default=list), ref) + assert orjson.dumps(obj, default=list) == ref def test_default_func_nested_str(self): """ @@ -139,10 +144,9 @@ def test_default_func_nested_str(self): def default(obj): return str(obj) - self.assertEqual( - orjson.dumps({"a": ref}, default=default), - b'{"a":"%s"}' % str(ref).encode("utf-8"), - ) + assert orjson.dumps({"a": ref}, default=default) == b'{"a":"%s"}' % str( + ref, + ).encode("utf-8") def test_default_func_list(self): """ @@ -154,10 +158,9 @@ def default(obj): if isinstance(obj, Custom): return [str(obj)] - self.assertEqual( - orjson.dumps({"a": ref}, default=default), - b'{"a":["%s"]}' % str(ref).encode("utf-8"), - ) + assert orjson.dumps({"a": ref}, default=default) == b'{"a":["%s"]}' % str( + ref, + ).encode("utf-8") def test_default_func_nested_list(self): """ @@ -168,9 +171,8 @@ def test_default_func_nested_list(self): def default(obj): return str(obj) - self.assertEqual( - orjson.dumps([ref] * 100, default=default), - b"[%s]" % b",".join(b'"%s"' % str(ref).encode("utf-8") for _ in range(100)), + assert orjson.dumps([ref] * 100, default=default) == b"[%s]" % b",".join( + b'"%s"' % str(ref).encode("utf-8") for _ in range(100) ) def test_default_func_bytes(self): @@ -182,17 +184,17 @@ def test_default_func_bytes(self): def default(obj): return bytes(obj) - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(ref, default=default) ran = False try: orjson.dumps(ref, default=default) except Exception as err: - self.assertIsInstance(err, orjson.JSONEncodeError) - self.assertEqual(str(err), "Type is not JSON serializable: Custom") + assert isinstance(err, orjson.JSONEncodeError) + assert str(err) == "Type is not JSON serializable: Custom" ran = True - self.assertTrue(ran) + assert ran def test_default_func_invalid_str(self): """ @@ -203,7 +205,7 @@ def test_default_func_invalid_str(self): def default(obj): return "\ud800" - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(ref, default=default) def test_default_lambda_ok(self): @@ -211,9 +213,8 @@ def test_default_lambda_ok(self): dumps() default lambda """ ref = Custom() - self.assertEqual( - orjson.dumps(ref, default=lambda x: str(x)), - b'"%s"' % str(ref).encode("utf-8"), + assert orjson.dumps(ref, default=str) == b'"%s"' % str(ref).encode( + "utf-8", ) def test_default_callable_ok(self): @@ -233,24 +234,24 @@ def __call__(self, obj): ref_obj = Custom() ref_bytes = b'"%s"' % str(ref_obj).encode("utf-8") for obj in [ref_obj] * 100: - self.assertEqual(orjson.dumps(obj, default=CustomSerializer()), ref_bytes) + assert orjson.dumps(obj, default=CustomSerializer()) == ref_bytes def test_default_recursion(self): """ dumps() default recursion limit """ - self.assertEqual(orjson.dumps(Recursive(254), default=default_recursive), b"0") + assert orjson.dumps(Recursive(254), default=default_recursive) == b"0" def test_default_recursion_reset(self): """ dumps() default recursion limit reset """ - self.assertEqual( + assert ( orjson.dumps( [Recursive(254), {"a": "b"}, Recursive(254), Recursive(254)], default=default_recursive, - ), - b'[0,{"a":"b"},0,0]', + ) + == b'[0,{"a":"b"},0,0]' ) def test_default_recursion_infinite(self): @@ -262,5 +263,91 @@ def test_default_recursion_infinite(self): def default(obj): return obj - with self.assertRaises(orjson.JSONEncodeError): + if SUPPORTS_GETREFCOUNT: + refcount = sys.getrefcount(ref) + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(ref, default=default) + if SUPPORTS_GETREFCOUNT: + assert sys.getrefcount(ref) == refcount + + def test_reference_cleanup_default_custom_pass(self): + ref = Custom() + + def default(obj): + if isinstance(ref, Custom): + return str(ref) + raise TypeError + + if SUPPORTS_GETREFCOUNT: + refcount = sys.getrefcount(ref) + orjson.dumps(ref, default=default) + if SUPPORTS_GETREFCOUNT: + assert sys.getrefcount(ref) == refcount + + def test_reference_cleanup_default_custom_error(self): + """ + references to encoded objects are cleaned up + """ + ref = Custom() + + def default(obj): + raise TypeError + + if SUPPORTS_GETREFCOUNT: + refcount = sys.getrefcount(ref) + with pytest.raises(orjson.JSONEncodeError): + orjson.dumps(ref, default=default) + if SUPPORTS_GETREFCOUNT: + assert sys.getrefcount(ref) == refcount + + def test_reference_cleanup_default_subclass(self): + ref = datetime.datetime(1970, 1, 1, 0, 0, 0) + + def default(obj): + if isinstance(ref, datetime.datetime): + return repr(ref) + raise TypeError + + if SUPPORTS_GETREFCOUNT: + refcount = sys.getrefcount(ref) + orjson.dumps(ref, option=orjson.OPT_PASSTHROUGH_DATETIME, default=default) + if SUPPORTS_GETREFCOUNT: + assert sys.getrefcount(ref) == refcount + + def test_reference_cleanup_default_subclass_lambda(self): + ref = uuid.uuid4() + + if SUPPORTS_GETREFCOUNT: + refcount = sys.getrefcount(ref) + orjson.dumps( + ref, + option=orjson.OPT_PASSTHROUGH_DATETIME, + default=str, + ) + if SUPPORTS_GETREFCOUNT: + assert sys.getrefcount(ref) == refcount + + @pytest.mark.skipif(numpy is None, reason="numpy is not installed") + def test_default_numpy(self): + ref = numpy.array([""] * 100) # type: ignore + if SUPPORTS_GETREFCOUNT: + refcount = sys.getrefcount(ref) + orjson.dumps( + ref, + option=orjson.OPT_SERIALIZE_NUMPY, + default=lambda val: val.tolist(), + ) + if SUPPORTS_GETREFCOUNT: + assert sys.getrefcount(ref) == refcount + + def test_default_set(self): + """ + dumps() default function with set + """ + + def default(obj): + if isinstance(obj, set): + return list(obj) + raise TypeError + + assert orjson.dumps({1, 2}, default=default) == b"[1,2]" diff --git a/test/test_dict.py b/test/test_dict.py index 7ecea329..a8e2c837 100644 --- a/test/test_dict.py +++ b/test/test_dict.py @@ -1,28 +1,170 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2018-2025), J. Nick Koston (2022), Anders Kaseorg (2022) -import unittest +import pytest import orjson -class DictTests(unittest.TestCase): +class TestDict: + def test_dict(self): + """ + dict + """ + obj = {"key": "value"} + ref = '{"key":"value"}' + assert orjson.dumps(obj) == ref.encode("utf-8") + assert orjson.loads(ref) == obj + + def test_dict_duplicate_loads(self): + assert orjson.loads(b'{"1":true,"1":false}') == {"1": False} + + def test_dict_empty(self): + obj = [{"key": [{}] * 4096}] * 4096 # type:ignore + assert orjson.loads(orjson.dumps(obj)) == obj + + def test_dict_large_dict(self): + """ + dict with >512 keys + """ + obj = {f"key_{idx}": [{}, {"a": [{}, {}, {}]}, {}] for idx in range(513)} # type: ignore + assert len(obj) == 513 + assert orjson.loads(orjson.dumps(obj)) == obj + + def test_dict_large_4096(self): + """ + dict with >4096 keys + """ + obj = {f"key_{idx}": f"value_{idx}" for idx in range(4097)} + assert len(obj) == 4097 + assert orjson.loads(orjson.dumps(obj)) == obj + + def test_dict_large_65536(self): + """ + dict with >65536 keys + """ + obj = {f"key_{idx}": f"value_{idx}" for idx in range(65537)} + assert len(obj) == 65537 + assert orjson.loads(orjson.dumps(obj)) == obj + + def test_dict_large_keys(self): + """ + dict with keys too large to cache + """ + obj = { + "keeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeey": "value", + } + ref = '{"keeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeey":"value"}' + assert orjson.dumps(obj) == ref.encode("utf-8") + assert orjson.loads(ref) == obj + + def test_dict_unicode(self): + """ + dict unicode keys + """ + obj = {"🐈": "value"} + ref = b'{"\xf0\x9f\x90\x88":"value"}' + assert orjson.dumps(obj) == ref + assert orjson.loads(ref) == obj + assert orjson.loads(ref)["🐈"] == "value" + + def test_dict_invalid_key_dumps(self): + """ + dict invalid key dumps() + """ + with pytest.raises(orjson.JSONEncodeError): + orjson.dumps({1: "value"}) + with pytest.raises(orjson.JSONEncodeError): + orjson.dumps({b"key": "value"}) + + def test_dict_invalid_key_loads(self): + """ + dict invalid key loads() + """ + with pytest.raises(orjson.JSONDecodeError): + orjson.loads('{1:"value"}') + with pytest.raises(orjson.JSONDecodeError): + orjson.loads('{{"a":true}:true}') + + def test_dict_similar_keys(self): + """ + loads() similar keys + + This was a regression in 3.4.2 caused by using + the implementation in wy instead of wyhash. + """ + assert orjson.loads( + '{"cf_status_firefox67": "---", "cf_status_firefox57": "verified"}', + ) == {"cf_status_firefox57": "verified", "cf_status_firefox67": "---"} + def test_dict_pop_replace_first(self): - """Test pop and replace a first key in a dict with other keys.""" + "Test pop and replace a first key in a dict with other keys." data = {"id": "any", "other": "any"} data.pop("id") + assert orjson.dumps(data) == b'{"other":"any"}' data["id"] = "new" - self.assertEqual(orjson.dumps(data), b'{"other":"any","id":"new"}') + assert orjson.dumps(data) == b'{"other":"any","id":"new"}' def test_dict_pop_replace_last(self): - """Test pop and replace a last key in a dict with other keys.""" + "Test pop and replace a last key in a dict with other keys." data = {"other": "any", "id": "any"} data.pop("id") + assert orjson.dumps(data) == b'{"other":"any"}' data["id"] = "new" - self.assertEqual(orjson.dumps(data), b'{"other":"any","id":"new"}') + assert orjson.dumps(data) == b'{"other":"any","id":"new"}' def test_dict_pop(self): - """Test pop and replace a key in a dict with no other keys.""" + "Test pop and replace a key in a dict with no other keys." data = {"id": "any"} data.pop("id") + assert orjson.dumps(data) == b"{}" data["id"] = "new" - self.assertEqual(orjson.dumps(data), b'{"id":"new"}') + assert orjson.dumps(data) == b'{"id":"new"}' + + def test_in_place(self): + "Mutate dict in-place" + data = {"id": "any", "static": "msg"} + data["id"] = "new" + assert orjson.dumps(data) == b'{"id":"new","static":"msg"}' + + def test_dict_0xff(self): + "dk_size <= 0xff" + data = {str(idx): idx for idx in range(0xFF)} + data.pop("112") + data["112"] = 1 + data["113"] = 2 + assert orjson.loads(orjson.dumps(data)) == data + + def test_dict_0xff_repeated(self): + "dk_size <= 0xff repeated" + for _ in range(100): + data = {str(idx): idx for idx in range(0xFF)} + data.pop("112") + data["112"] = 1 + data["113"] = 2 + assert orjson.loads(orjson.dumps(data)) == data + + def test_dict_0xffff(self): + "dk_size <= 0xffff" + data = {str(idx): idx for idx in range(0xFFFF)} + data.pop("112") + data["112"] = 1 + data["113"] = 2 + assert orjson.loads(orjson.dumps(data)) == data + + def test_dict_0xffff_repeated(self): + "dk_size <= 0xffff repeated" + for _ in range(100): + data = {str(idx): idx for idx in range(0xFFFF)} + data.pop("112") + data["112"] = 1 + data["113"] = 2 + assert orjson.loads(orjson.dumps(data)) == data + + def test_dict_dict(self): + class C: + def __init__(self): + self.a = 0 + self.b = 1 + + assert orjson.dumps(C().__dict__) == b'{"a":0,"b":1}' diff --git a/test/test_enum.py b/test/test_enum.py index 216e94aa..390bfef3 100644 --- a/test/test_enum.py +++ b/test/test_enum.py @@ -1,8 +1,10 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2020-2025) import datetime import enum -import unittest + +import pytest import orjson @@ -50,69 +52,70 @@ class UnspecifiedEnum(enum.Enum): A = "a" B = 1 C = FloatEnum.ONE - D = {"d": IntEnum.ONE} + D = {"d": IntEnum.ONE} # noqa: RUF012 E = Custom("c") F = datetime.datetime(1970, 1, 1) -class EnumTests(unittest.TestCase): +class TestEnum: def test_cannot_subclass(self): """ enum.Enum cannot be subclassed obj->ob_type->ob_base will always be enum.EnumMeta """ - with self.assertRaises(TypeError): + with pytest.raises(TypeError): class Subclass(StrEnum): # type: ignore B = "b" def test_arbitrary_enum(self): - self.assertEqual(orjson.dumps(UnspecifiedEnum.A), b'"a"') - self.assertEqual(orjson.dumps(UnspecifiedEnum.B), b"1") - self.assertEqual(orjson.dumps(UnspecifiedEnum.C), b"1.1") - self.assertEqual(orjson.dumps(UnspecifiedEnum.D), b'{"d":1}') + assert orjson.dumps(UnspecifiedEnum.A) == b'"a"' + assert orjson.dumps(UnspecifiedEnum.B) == b"1" + assert orjson.dumps(UnspecifiedEnum.C) == b"1.1" + assert orjson.dumps(UnspecifiedEnum.D) == b'{"d":1}' def test_custom_enum(self): - self.assertEqual(orjson.dumps(UnspecifiedEnum.E, default=default), b'"c"') + assert orjson.dumps(UnspecifiedEnum.E, default=default) == b'"c"' def test_enum_options(self): - self.assertEqual( - orjson.dumps(UnspecifiedEnum.F, option=orjson.OPT_NAIVE_UTC), - b'"1970-01-01T00:00:00+00:00"', + assert ( + orjson.dumps(UnspecifiedEnum.F, option=orjson.OPT_NAIVE_UTC) + == b'"1970-01-01T00:00:00+00:00"' ) def test_int_enum(self): - self.assertEqual(orjson.dumps(IntEnum.ONE), b"1") + assert orjson.dumps(IntEnum.ONE) == b"1" def test_intenum_enum(self): - self.assertEqual(orjson.dumps(IntEnumEnum.ONE), b"1") + assert orjson.dumps(IntEnumEnum.ONE) == b"1" def test_intflag_enum(self): - self.assertEqual(orjson.dumps(IntFlagEnum.ONE), b"1") + assert orjson.dumps(IntFlagEnum.ONE) == b"1" def test_flag_enum(self): - self.assertEqual(orjson.dumps(FlagEnum.ONE), b"1") + assert orjson.dumps(FlagEnum.ONE) == b"1" def test_auto_enum(self): - self.assertEqual(orjson.dumps(AutoEnum.A), b'"a"') + assert orjson.dumps(AutoEnum.A) == b'"a"' def test_float_enum(self): - self.assertEqual(orjson.dumps(FloatEnum.ONE), b"1.1") + assert orjson.dumps(FloatEnum.ONE) == b"1.1" def test_str_enum(self): - self.assertEqual(orjson.dumps(StrEnum.AAA), b'"aaa"') + assert orjson.dumps(StrEnum.AAA) == b'"aaa"' def test_bool_enum(self): - with self.assertRaises(TypeError): + with pytest.raises(TypeError): - class BoolEnum(bool, enum.Enum): + class BoolEnum(bool, enum.Enum): # type: ignore TRUE = True def test_non_str_keys_enum(self): - self.assertEqual( - orjson.dumps({StrEnum.AAA: 1}, option=orjson.OPT_NON_STR_KEYS), b'{"aaa":1}' + assert ( + orjson.dumps({StrEnum.AAA: 1}, option=orjson.OPT_NON_STR_KEYS) + == b'{"aaa":1}' ) - self.assertEqual( - orjson.dumps({IntEnum.ONE: 1}, option=orjson.OPT_NON_STR_KEYS), b'{"1":1}' + assert ( + orjson.dumps({IntEnum.ONE: 1}, option=orjson.OPT_NON_STR_KEYS) == b'{"1":1}' ) diff --git a/test/test_error.py b/test/test_error.py index 63c6cebe..e3c83d8c 100644 --- a/test/test_error.py +++ b/test/test_error.py @@ -1,13 +1,13 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2021-2025), Eric Jolibois (2021), o.ermakov (2023) import json -import unittest import pytest import orjson -from .util import read_fixture_str +from .util import needs_data, read_fixture_str ASCII_TEST = b"""\ { @@ -24,7 +24,7 @@ """ -class JsonDecodeErrorTests(unittest.TestCase): +class TestJsonDecodeError: def _get_error_infos(self, json_decode_error_exc_info): return { k: v @@ -45,6 +45,13 @@ def _test(self, data, expected_err_infos): == expected_err_infos ) + def test_empty(self): + with pytest.raises(orjson.JSONDecodeError) as json_exc_info: + orjson.loads("") + assert str(json_exc_info.value).startswith( + "Input is a zero-length, empty document:", + ) + def test_ascii(self): self._test( ASCII_TEST, @@ -75,6 +82,7 @@ def test_four_byte(self): {"pos": 19, "lineno": 4, "colno": 1}, ) + @needs_data def test_tab(self): data = read_fixture_str("fail26.json", "jsonchecker") with pytest.raises(json.decoder.JSONDecodeError) as json_exc_info: @@ -86,11 +94,99 @@ def test_tab(self): "colno": 6, } - with pytest.raises(json.decoder.JSONDecodeError) as orjson_exc_info: + with pytest.raises(json.decoder.JSONDecodeError) as json_exc_info: orjson.loads(data) - assert self._get_error_infos(orjson_exc_info) == { + assert self._get_error_infos(json_exc_info) == { "pos": 6, "lineno": 1, "colno": 7, } + + +class Custom: + pass + + +class CustomException(Exception): + pass + + +def default_typeerror(obj): + raise TypeError + + +def default_notimplementederror(obj): + raise NotImplementedError + + +def default_systemerror(obj): + raise SystemError + + +def default_importerror(obj): + import doesnotexist # noqa: PLC0415 + + assert doesnotexist + + +CUSTOM_ERROR_MESSAGE = "zxc" + + +def default_customerror(obj): + raise CustomException(CUSTOM_ERROR_MESSAGE) + + +class TestJsonEncodeError: + def test_dumps_arg(self): + with pytest.raises(orjson.JSONEncodeError) as exc_info: + orjson.dumps() # type: ignore + assert exc_info.type == orjson.JSONEncodeError + assert ( + str(exc_info.value) + == "dumps() missing 1 required positional argument: 'obj'" + ) + assert exc_info.value.__cause__ is None + + def test_dumps_chain_none(self): + with pytest.raises(orjson.JSONEncodeError) as exc_info: + orjson.dumps(Custom()) + assert exc_info.type == orjson.JSONEncodeError + assert str(exc_info.value) == "Type is not JSON serializable: Custom" + assert exc_info.value.__cause__ is None + + def test_dumps_chain_u64(self): + with pytest.raises(orjson.JSONEncodeError) as exc_info: + orjson.dumps([18446744073709551615, Custom()]) + assert exc_info.type == orjson.JSONEncodeError + assert exc_info.value.__cause__ is None + + def test_dumps_chain_default_typeerror(self): + with pytest.raises(orjson.JSONEncodeError) as exc_info: + orjson.dumps(Custom(), default=default_typeerror) + assert exc_info.type == orjson.JSONEncodeError + assert isinstance(exc_info.value.__cause__, TypeError) + + def test_dumps_chain_default_systemerror(self): + with pytest.raises(orjson.JSONEncodeError) as exc_info: + orjson.dumps(Custom(), default=default_systemerror) + assert exc_info.type == orjson.JSONEncodeError + assert isinstance(exc_info.value.__cause__, SystemError) + + def test_dumps_chain_default_importerror(self): + with pytest.raises(orjson.JSONEncodeError) as exc_info: + orjson.dumps(Custom(), default=default_importerror) + assert exc_info.type == orjson.JSONEncodeError + assert isinstance(exc_info.value.__cause__, ImportError) + + def test_dumps_chain_default_customerror(self): + with pytest.raises(orjson.JSONEncodeError) as exc_info: + orjson.dumps(Custom(), default=default_customerror) + assert exc_info.type == orjson.JSONEncodeError + assert isinstance(exc_info.value.__cause__, CustomException) + assert str(exc_info.value.__cause__) == CUSTOM_ERROR_MESSAGE + + def test_dumps_normalize_exception(self): + with pytest.raises(orjson.JSONEncodeError) as exc_info: + orjson.dumps(10**60) + assert exc_info.type == orjson.JSONEncodeError diff --git a/test/test_escape.py b/test/test_escape.py new file mode 100644 index 00000000..31376921 --- /dev/null +++ b/test/test_escape.py @@ -0,0 +1,127 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2025) + +import orjson + + +def test_issue565(): + assert ( + orjson.dumps("\n\r\u000b\f\u001c\u001d\u001e") + == b'"\\n\\r\\u000b\\f\\u001c\\u001d\\u001e"' + ) + + +def test_0x00(): + assert orjson.dumps("\u0000") == b'"\\u0000"' + + +def test_0x01(): + assert orjson.dumps("\u0001") == b'"\\u0001"' + + +def test_0x02(): + assert orjson.dumps("\u0002") == b'"\\u0002"' + + +def test_0x03(): + assert orjson.dumps("\u0003") == b'"\\u0003"' + + +def test_0x04(): + assert orjson.dumps("\u0004") == b'"\\u0004"' + + +def test_0x05(): + assert orjson.dumps("\u0005") == b'"\\u0005"' + + +def test_0x06(): + assert orjson.dumps("\u0006") == b'"\\u0006"' + + +def test_0x07(): + assert orjson.dumps("\u0007") == b'"\\u0007"' + + +def test_0x08(): + assert orjson.dumps("\u0008") == b'"\\b"' + + +def test_0x09(): + assert orjson.dumps("\u0009") == b'"\\t"' + + +def test_0x0a(): + assert orjson.dumps("\u000a") == b'"\\n"' + + +def test_0x0b(): + assert orjson.dumps("\u000b") == b'"\\u000b"' + + +def test_0x0c(): + assert orjson.dumps("\u000c") == b'"\\f"' + + +def test_0x0d(): + assert orjson.dumps("\u000d") == b'"\\r"' + + +def test_0x0e(): + assert orjson.dumps("\u000e") == b'"\\u000e"' + + +def test_0x0f(): + assert orjson.dumps("\u000f") == b'"\\u000f"' + + +def test_0x10(): + assert orjson.dumps("\u0010") == b'"\\u0010"' + + +def test_0x11(): + assert orjson.dumps("\u0011") == b'"\\u0011"' + + +def test_0x12(): + assert orjson.dumps("\u0012") == b'"\\u0012"' + + +def test_0x13(): + assert orjson.dumps("\u0013") == b'"\\u0013"' + + +def test_0x14(): + assert orjson.dumps("\u0014") == b'"\\u0014"' + + +def test_0x15(): + assert orjson.dumps("\u0015") == b'"\\u0015"' + + +def test_0x16(): + assert orjson.dumps("\u0016") == b'"\\u0016"' + + +def test_0x17(): + assert orjson.dumps("\u0017") == b'"\\u0017"' + + +def test_0x18(): + assert orjson.dumps("\u0018") == b'"\\u0018"' + + +def test_0x19(): + assert orjson.dumps("\u0019") == b'"\\u0019"' + + +def test_0x1a(): + assert orjson.dumps("\u001a") == b'"\\u001a"' + + +def test_backslash(): + assert orjson.dumps("\\") == b'"\\\\"' + + +def test_quote(): + assert orjson.dumps('"') == b'"\\""' diff --git a/test/test_fake.py b/test/test_fake.py new file mode 100644 index 00000000..c6806816 --- /dev/null +++ b/test/test_fake.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2023-2025) + +import random + +import pytest + +import orjson + +try: + from faker import Faker +except ImportError: + Faker = None # type: ignore + +NUM_LOOPS = 10 +NUM_SHUFFLES = 10 +NUM_ENTRIES = 250 + +FAKER_LOCALES = [ + "ar_AA", + "fi_FI", + "fil_PH", + "he_IL", + "ja_JP", + "th_TH", + "tr_TR", + "uk_UA", + "vi_VN", +] + + +class TestFaker: + @pytest.mark.skipif(Faker is None, reason="faker not available") + def test_faker(self): + fake = Faker(FAKER_LOCALES) + profile_keys = list( + set(fake.profile().keys()) - {"birthdate", "current_location"}, + ) + for _ in range(NUM_LOOPS): + data = [ + { + "person": fake.profile(profile_keys), + "emoji": fake.emoji(), + "text": fake.paragraphs(), + } + for _ in range(NUM_ENTRIES) + ] + for _ in range(NUM_SHUFFLES): + random.shuffle(data) + output = orjson.dumps(data) + assert orjson.loads(output) == data diff --git a/test/test_fixture.py b/test/test_fixture.py index 5eeca468..798a7bf7 100644 --- a/test/test_fixture.py +++ b/test/test_fixture.py @@ -1,28 +1,31 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2018-2025) -import unittest +import pytest import orjson -from .util import read_fixture_bytes, read_fixture_str +from .util import needs_data, read_fixture_bytes, read_fixture_str -class FixtureTests(unittest.TestCase): +@needs_data +class TestFixture: def test_twitter(self): """ loads(),dumps() twitter.json """ val = read_fixture_str("twitter.json.xz") read = orjson.loads(val) - orjson.dumps(read) + assert orjson.loads(orjson.dumps(read)) == read + @needs_data def test_canada(self): """ loads(), dumps() canada.json """ val = read_fixture_str("canada.json.xz") read = orjson.loads(val) - orjson.dumps(read) + assert orjson.loads(orjson.dumps(read)) == read def test_citm_catalog(self): """ @@ -30,7 +33,7 @@ def test_citm_catalog(self): """ val = read_fixture_str("citm_catalog.json.xz") read = orjson.loads(val) - orjson.dumps(read) + assert orjson.loads(orjson.dumps(read)) == read def test_github(self): """ @@ -38,7 +41,7 @@ def test_github(self): """ val = read_fixture_str("github.json.xz") read = orjson.loads(val) - orjson.dumps(read) + assert orjson.loads(orjson.dumps(read)) == read def test_blns(self): """ @@ -49,5 +52,5 @@ def test_blns(self): val = read_fixture_bytes("blns.txt.xz") for line in val.split(b"\n"): if line and not line.startswith(b"#"): - with self.assertRaises(orjson.JSONDecodeError): + with pytest.raises(orjson.JSONDecodeError): _ = orjson.loads(b'"' + val + b'"') diff --git a/test/test_fragment.py b/test/test_fragment.py new file mode 100644 index 00000000..2a7926b6 --- /dev/null +++ b/test/test_fragment.py @@ -0,0 +1,1061 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2023-2025) + +import pytest + +import orjson + +try: + import pandas as pd +except ImportError: + pd = None # type: ignore + +from .util import needs_data, read_fixture_bytes + + +class TestFragment: + def test_fragment_fragment_eq(self): + assert orjson.Fragment(b"{}") != orjson.Fragment(b"{}") + + def test_fragment_fragment_not_mut(self): + fragment = orjson.Fragment(b"{}") + with pytest.raises(AttributeError): + fragment.contents = b"[]" + assert orjson.dumps(fragment) == b"{}" + + def test_fragment_repr(self): + assert repr(orjson.Fragment(b"{}")).startswith("?","hex":"ģ䕧覫췯ꯍ\uef4a","true":true,"false":false,"null":null,"array":[],"object":{},"address":"50 St. James Street","url":"http://www.JSON.org/","comment":"// /* */":" "," s p a c e d ":[1,2,3,4,5,6,7],"compact":[1,2,3,4,5,6,7],"jsontext":"{\\"object with 1 member\\":[\\"array with 1 element\\"]}","quotes":"" \\" %22 0x22 034 "","/\\\\\\"쫾몾ꮘﳞ볚\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?":"A key can be any string"},0.5,98.6,99.44,1066,10.0,1.0,0.1,1.0,2.0,2.0,"rosebud"]'.encode() +PATTERN_1 = '["JSON Test Pattern pass1",{"object with 1 member":["array with 1 element"]},{},[],-42,true,false,null,{"integer":1234567890,"real":-9876.54321,"e":1.23456789e-13,"E":1.23456789e+34,"":2.3456789012e+76,"zero":0,"one":1,"space":" ","quote":"\\"","backslash":"\\\\","controls":"\\b\\f\\n\\r\\t","slash":"/ & /","alpha":"abcdefghijklmnopqrstuvwyz","ALPHA":"ABCDEFGHIJKLMNOPQRSTUVWYZ","digit":"0123456789","0123456789":"digit","special":"`1~!@#$%^&*()_+-={\':[,]}|;.?","hex":"ģ䕧覫췯ꯍ\uef4a","true":true,"false":false,"null":null,"array":[],"object":{},"address":"50 St. James Street","url":"http://www.JSON.org/","comment":"// /* */":" "," s p a c e d ":[1,2,3,4,5,6,7],"compact":[1,2,3,4,5,6,7],"jsontext":"{\\"object with 1 member\\":[\\"array with 1 element\\"]}","quotes":"" \\" %22 0x22 034 "","/\\\\\\"쫾몾ꮘﳞ볚\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?":"A key can be any string"},0.5,98.6,99.44,1066,10.0,1.0,0.1,1.0,2.0,2.0,"rosebud"]'.encode() -class JsonCheckerTests(unittest.TestCase): +@needs_data +class TestJsonChecker: def _run_fail_json(self, filename, exc=orjson.JSONDecodeError): data = read_fixture_str(filename, "jsonchecker") - self.assertRaises(exc, orjson.loads, data) + pytest.raises(exc, orjson.loads, data) def _run_pass_json(self, filename, match=""): data = read_fixture_str(filename, "jsonchecker") - self.assertEqual(orjson.dumps(orjson.loads(data)), match) + assert orjson.dumps(orjson.loads(data)) == match def test_fail01(self): """ @@ -131,7 +131,8 @@ def test_fail18(self): fail18.json """ self._run_pass_json( - "fail18.json", b'[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]' + "fail18.json", + b'[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', ) def test_fail19(self): @@ -235,7 +236,8 @@ def test_pass02(self): pass02.json """ self._run_pass_json( - "pass02.json", b'[[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]]' + "pass02.json", + b'[[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]]', ) def test_pass03(self): diff --git a/test/test_memory.py b/test/test_memory.py index 5cf4a2ea..b5328ac5 100644 --- a/test/test_memory.py +++ b/test/test_memory.py @@ -1,11 +1,12 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2019-2026), Rami Chowdhury (2020) import dataclasses import datetime import gc import random -import unittest -from typing import List + +from .util import SUPPORTS_MEMORYVIEW, numpy, pandas try: import pytz @@ -16,14 +17,12 @@ import psutil except ImportError: psutil = None # type: ignore + import pytest import orjson -try: - import numpy -except ImportError: - numpy = None # type: ignore +from .util import IS_FREETHREADING FIXTURE = '{"a":[81891289, 8919812.190129012], "b": false, "c": null, "d": "東京"}' @@ -43,7 +42,7 @@ class Object: id: int updated_at: datetime.datetime name: str - members: List[Member] + members: list[Member] DATACLASS_FIXTURE = [ @@ -52,22 +51,23 @@ class Object: datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=random.randint(0, 10000)), str(i) * 3, - [Member(j, True) for j in range(0, 10)], + [Member(j, True) for j in range(10)], ) for i in range(100000, 101000) ] -MAX_INCREASE = 1048576 # 1MiB +MAX_INCREASE = 4194304 # 4MiB + +if IS_FREETHREADING: + MAX_INCREASE *= 4 class Unsupported: pass -class MemoryTests(unittest.TestCase): - @pytest.mark.skipif( - psutil is None, reason="psutil install broken on win, python3.9, Azure" - ) +class TestMemory: + @pytest.mark.skipif(psutil is None, reason="psutil not installed") def test_memory_loads(self): """ loads() memory leak @@ -75,15 +75,16 @@ def test_memory_loads(self): proc = psutil.Process() gc.collect() val = orjson.loads(FIXTURE) + assert val mem = proc.memory_info().rss for _ in range(10000): val = orjson.loads(FIXTURE) + assert val gc.collect() - self.assertTrue(proc.memory_info().rss <= mem + MAX_INCREASE) + assert proc.memory_info().rss <= mem + MAX_INCREASE - @pytest.mark.skipif( - psutil is None, reason="psutil install broken on win, python3.9, Azure" - ) + @pytest.mark.skipif(psutil is None, reason="psutil not installed") + @pytest.mark.skipif(SUPPORTS_MEMORYVIEW is False, reason="memoryview") def test_memory_loads_memoryview(self): """ loads() memory leak using memoryview @@ -92,15 +93,15 @@ def test_memory_loads_memoryview(self): gc.collect() fixture = FIXTURE.encode("utf-8") val = orjson.loads(fixture) + assert val mem = proc.memory_info().rss for _ in range(10000): val = orjson.loads(memoryview(fixture)) + assert val gc.collect() - self.assertTrue(proc.memory_info().rss <= mem + MAX_INCREASE) + assert proc.memory_info().rss <= mem + MAX_INCREASE - @pytest.mark.skipif( - psutil is None, reason="psutil install broken on win, python3.9, Azure" - ) + @pytest.mark.skipif(psutil is None, reason="psutil not installed") def test_memory_dumps(self): """ dumps() memory leak @@ -109,15 +110,16 @@ def test_memory_dumps(self): gc.collect() fixture = orjson.loads(FIXTURE) val = orjson.dumps(fixture) + assert val mem = proc.memory_info().rss for _ in range(10000): val = orjson.dumps(fixture) + assert val gc.collect() - self.assertTrue(proc.memory_info().rss <= mem + MAX_INCREASE) + assert proc.memory_info().rss <= mem + MAX_INCREASE + assert proc.memory_info().rss <= mem + MAX_INCREASE - @pytest.mark.skipif( - psutil is None, reason="psutil install broken on win, python3.9, Azure" - ) + @pytest.mark.skipif(psutil is None, reason="psutil not installed") def test_memory_loads_exc(self): """ loads() memory leak exception without a GC pause @@ -133,12 +135,10 @@ def test_memory_loads_exc(self): except orjson.JSONDecodeError: i += 1 assert n == i - self.assertTrue(proc.memory_info().rss <= mem + MAX_INCREASE) + assert proc.memory_info().rss <= mem + MAX_INCREASE gc.enable() - @pytest.mark.skipif( - psutil is None, reason="psutil install broken on win, python3.9, Azure" - ) + @pytest.mark.skipif(psutil is None, reason="psutil not installed") def test_memory_dumps_exc(self): """ dumps() memory leak exception without a GC pause @@ -155,12 +155,10 @@ def test_memory_dumps_exc(self): except orjson.JSONEncodeError: i += 1 assert n == i - self.assertTrue(proc.memory_info().rss <= mem + MAX_INCREASE) + assert proc.memory_info().rss <= mem + MAX_INCREASE gc.enable() - @pytest.mark.skipif( - psutil is None, reason="psutil install broken on win, python3.9, Azure" - ) + @pytest.mark.skipif(psutil is None, reason="psutil not installed") def test_memory_dumps_default(self): """ dumps() default memory leak @@ -181,12 +179,11 @@ def __str__(self): mem = proc.memory_info().rss for _ in range(10000): val = orjson.dumps(fixture, default=default) + assert val gc.collect() - self.assertTrue(proc.memory_info().rss <= mem + MAX_INCREASE) + assert proc.memory_info().rss <= mem + MAX_INCREASE - @pytest.mark.skipif( - psutil is None, reason="psutil install broken on win, python3.9, Azure" - ) + @pytest.mark.skipif(psutil is None, reason="psutil not installed") def test_memory_dumps_dataclass(self): """ dumps() dataclass memory leak @@ -194,15 +191,18 @@ def test_memory_dumps_dataclass(self): proc = psutil.Process() gc.collect() val = orjson.dumps(DATACLASS_FIXTURE) + assert val mem = proc.memory_info().rss for _ in range(100): val = orjson.dumps(DATACLASS_FIXTURE) + assert val + assert val gc.collect() - self.assertTrue(proc.memory_info().rss <= mem + MAX_INCREASE) + assert proc.memory_info().rss <= mem + MAX_INCREASE @pytest.mark.skipif( psutil is None or pytz is None, - reason="psutil install broken on win, python3.9, Azure", + reason="psutil not installed", ) def test_memory_dumps_pytz_tzinfo(self): """ @@ -212,45 +212,82 @@ def test_memory_dumps_pytz_tzinfo(self): gc.collect() dt = datetime.datetime.now() val = orjson.dumps(pytz.UTC.localize(dt)) + assert val mem = proc.memory_info().rss for _ in range(50000): val = orjson.dumps(pytz.UTC.localize(dt)) + assert val + assert val gc.collect() - self.assertTrue(proc.memory_info().rss <= mem + MAX_INCREASE) + assert proc.memory_info().rss <= mem + MAX_INCREASE - @pytest.mark.skipif( - psutil is None, reason="psutil install broken on win, python3.9, Azure" - ) + @pytest.mark.skipif(psutil is None, reason="psutil not installed") def test_memory_loads_keys(self): """ loads() memory leak with number of keys causing cache eviction """ proc = psutil.Process() gc.collect() - fixture = {"key_%s" % idx: "value" for idx in range(1024)} - self.assertEqual(len(fixture), 1024) + fixture = {f"key_{idx}": "value" for idx in range(1024)} + assert len(fixture) == 1024 val = orjson.dumps(fixture) loaded = orjson.loads(val) + assert loaded mem = proc.memory_info().rss for _ in range(100): loaded = orjson.loads(val) + assert loaded gc.collect() - self.assertTrue(proc.memory_info().rss <= mem + MAX_INCREASE) + assert proc.memory_info().rss <= mem + MAX_INCREASE - @pytest.mark.skipif( - psutil is None, reason="psutil install broken on win, python3.9, Azure" - ) + @pytest.mark.skipif(psutil is None, reason="psutil not installed") @pytest.mark.skipif(numpy is None, reason="numpy is not installed") def test_memory_dumps_numpy(self): """ - dumps() dataclass memory leak + dumps() numpy memory leak """ proc = psutil.Process() gc.collect() - fixture = numpy.random.rand(4, 4, 4) + fixture = numpy.random.rand(4, 4, 4) # type: ignore val = orjson.dumps(fixture, option=orjson.OPT_SERIALIZE_NUMPY) + assert val mem = proc.memory_info().rss for _ in range(100): val = orjson.dumps(fixture, option=orjson.OPT_SERIALIZE_NUMPY) + assert val + assert val + gc.collect() + assert proc.memory_info().rss <= mem + MAX_INCREASE + + @pytest.mark.skipif(psutil is None, reason="psutil not installed") + @pytest.mark.skipif(pandas is None, reason="pandas is not installed") + def test_memory_dumps_pandas(self): + """ + dumps() pandas memory leak + """ + proc = psutil.Process() + gc.collect() + numpy.random.rand(4, 4, 4) # type: ignore + df = pandas.Series(numpy.random.rand(4, 4, 4).tolist()) # type: ignore + val = df.map(orjson.dumps) + assert not val.empty + mem = proc.memory_info().rss + for _ in range(100): + val = df.map(orjson.dumps) + assert not val.empty + gc.collect() + assert proc.memory_info().rss <= mem + MAX_INCREASE + + @pytest.mark.skipif(psutil is None, reason="psutil not installed") + def test_memory_dumps_fragment(self): + """ + dumps() Fragment memory leak + """ + proc = psutil.Process() + gc.collect() + orjson.dumps(orjson.Fragment(str(0))) + mem = proc.memory_info().rss + for i in range(10000): + orjson.dumps(orjson.Fragment(str(i))) gc.collect() - self.assertTrue(proc.memory_info().rss <= mem + MAX_INCREASE) + assert proc.memory_info().rss <= mem + MAX_INCREASE diff --git a/test/test_non_str_keys.py b/test/test_non_str_keys.py index 35ce37b7..b677f8e9 100644 --- a/test/test_non_str_keys.py +++ b/test/test_non_str_keys.py @@ -1,8 +1,8 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2020-2025) import dataclasses import datetime -import unittest import uuid import pytest @@ -14,212 +14,210 @@ except ImportError: pytz = None # type: ignore -try: - import numpy -except ImportError: - numpy = None # type: ignore +from .util import numpy class SubStr(str): pass -class NonStrKeyTests(unittest.TestCase): +class TestNonStrKeyTests: def test_dict_keys_duplicate(self): """ OPT_NON_STR_KEYS serializes duplicate keys """ - self.assertEqual( - orjson.dumps({"1": True, 1: False}, option=orjson.OPT_NON_STR_KEYS), - b'{"1":true,"1":false}', + assert ( + orjson.dumps({"1": True, 1: False}, option=orjson.OPT_NON_STR_KEYS) + == b'{"1":true,"1":false}' ) def test_dict_keys_int(self): - self.assertEqual( - orjson.dumps({1: True, 2: False}, option=orjson.OPT_NON_STR_KEYS), - b'{"1":true,"2":false}', + assert ( + orjson.dumps({1: True, 2: False}, option=orjson.OPT_NON_STR_KEYS) + == b'{"1":true,"2":false}' ) def test_dict_keys_substr(self): - self.assertEqual( - orjson.dumps({SubStr("aaa"): True}, option=orjson.OPT_NON_STR_KEYS), - b'{"aaa":true}', + assert ( + orjson.dumps({SubStr("aaa"): True}, option=orjson.OPT_NON_STR_KEYS) + == b'{"aaa":true}' ) def test_dict_keys_substr_passthrough(self): """ OPT_PASSTHROUGH_SUBCLASS does not affect OPT_NON_STR_KEYS """ - self.assertEqual( + assert ( orjson.dumps( {SubStr("aaa"): True}, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_PASSTHROUGH_SUBCLASS, - ), - b'{"aaa":true}', + ) + == b'{"aaa":true}' ) def test_dict_keys_substr_invalid(self): - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps({SubStr("\ud800"): True}, option=orjson.OPT_NON_STR_KEYS) def test_dict_keys_strict(self): """ OPT_NON_STR_KEYS does not respect OPT_STRICT_INTEGER """ - self.assertEqual( + assert ( orjson.dumps( {9223372036854775807: True}, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_STRICT_INTEGER, - ), - b'{"9223372036854775807":true}', + ) + == b'{"9223372036854775807":true}' ) def test_dict_keys_int_range_valid_i64(self): """ OPT_NON_STR_KEYS has a i64 range for int, valid """ - self.assertEqual( + assert ( orjson.dumps( {9223372036854775807: True}, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_STRICT_INTEGER, - ), - b'{"9223372036854775807":true}', + ) + == b'{"9223372036854775807":true}' ) - self.assertEqual( + assert ( orjson.dumps( {-9223372036854775807: True}, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_STRICT_INTEGER, - ), - b'{"-9223372036854775807":true}', + ) + == b'{"-9223372036854775807":true}' ) - self.assertEqual( + assert ( orjson.dumps( {9223372036854775809: True}, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_STRICT_INTEGER, - ), - b'{"9223372036854775809":true}', + ) + == b'{"9223372036854775809":true}' ) def test_dict_keys_int_range_valid_u64(self): """ OPT_NON_STR_KEYS has a u64 range for int, valid """ - self.assertEqual( + assert ( orjson.dumps( {0: True}, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_STRICT_INTEGER, - ), - b'{"0":true}', + ) + == b'{"0":true}' ) - self.assertEqual( + assert ( orjson.dumps( {18446744073709551615: True}, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_STRICT_INTEGER, - ), - b'{"18446744073709551615":true}', + ) + == b'{"18446744073709551615":true}' ) def test_dict_keys_int_range_invalid(self): """ OPT_NON_STR_KEYS has a range of i64::MIN to u64::MAX """ - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps({-9223372036854775809: True}, option=orjson.OPT_NON_STR_KEYS) - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps({18446744073709551616: True}, option=orjson.OPT_NON_STR_KEYS) def test_dict_keys_float(self): - self.assertEqual( - orjson.dumps({1.1: True, 2.2: False}, option=orjson.OPT_NON_STR_KEYS), - b'{"1.1":true,"2.2":false}', + assert ( + orjson.dumps({1.1: True, 2.2: False}, option=orjson.OPT_NON_STR_KEYS) + == b'{"1.1":true,"2.2":false}' ) def test_dict_keys_inf(self): - self.assertEqual( - orjson.dumps({float("Infinity"): True}, option=orjson.OPT_NON_STR_KEYS), - b'{"null":true}', + assert ( + orjson.dumps({float("Infinity"): True}, option=orjson.OPT_NON_STR_KEYS) + == b'{"null":true}' ) - self.assertEqual( - orjson.dumps({float("-Infinity"): True}, option=orjson.OPT_NON_STR_KEYS), - b'{"null":true}', + assert ( + orjson.dumps({float("-Infinity"): True}, option=orjson.OPT_NON_STR_KEYS) + == b'{"null":true}' ) def test_dict_keys_nan(self): - self.assertEqual( - orjson.dumps({float("NaN"): True}, option=orjson.OPT_NON_STR_KEYS), - b'{"null":true}', + assert ( + orjson.dumps({float("NaN"): True}, option=orjson.OPT_NON_STR_KEYS) + == b'{"null":true}' ) def test_dict_keys_bool(self): - self.assertEqual( - orjson.dumps({True: True, False: False}, option=orjson.OPT_NON_STR_KEYS), - b'{"true":true,"false":false}', + assert ( + orjson.dumps({True: True, False: False}, option=orjson.OPT_NON_STR_KEYS) + == b'{"true":true,"false":false}' ) def test_dict_keys_datetime(self): - self.assertEqual( + assert ( orjson.dumps( {datetime.datetime(2000, 1, 1, 2, 3, 4, 123): True}, option=orjson.OPT_NON_STR_KEYS, - ), - b'{"2000-01-01T02:03:04.000123":true}', + ) + == b'{"2000-01-01T02:03:04.000123":true}' ) def test_dict_keys_datetime_opt(self): - self.assertEqual( + assert ( orjson.dumps( {datetime.datetime(2000, 1, 1, 2, 3, 4, 123): True}, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_OMIT_MICROSECONDS | orjson.OPT_NAIVE_UTC | orjson.OPT_UTC_Z, - ), - b'{"2000-01-01T02:03:04Z":true}', + ) + == b'{"2000-01-01T02:03:04Z":true}' ) def test_dict_keys_datetime_passthrough(self): """ OPT_PASSTHROUGH_DATETIME does not affect OPT_NON_STR_KEYS """ - self.assertEqual( + assert ( orjson.dumps( {datetime.datetime(2000, 1, 1, 2, 3, 4, 123): True}, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_PASSTHROUGH_DATETIME, - ), - b'{"2000-01-01T02:03:04.000123":true}', + ) + == b'{"2000-01-01T02:03:04.000123":true}' ) def test_dict_keys_uuid(self): """ OPT_NON_STR_KEYS always serializes UUID as keys """ - self.assertEqual( + assert ( orjson.dumps( {uuid.UUID("7202d115-7ff3-4c81-a7c1-2a1f067b1ece"): True}, option=orjson.OPT_NON_STR_KEYS, - ), - b'{"7202d115-7ff3-4c81-a7c1-2a1f067b1ece":true}', + ) + == b'{"7202d115-7ff3-4c81-a7c1-2a1f067b1ece":true}' ) def test_dict_keys_date(self): - self.assertEqual( + assert ( orjson.dumps( - {datetime.date(1970, 1, 1): True}, option=orjson.OPT_NON_STR_KEYS - ), - b'{"1970-01-01":true}', + {datetime.date(1970, 1, 1): True}, + option=orjson.OPT_NON_STR_KEYS, + ) + == b'{"1970-01-01":true}' ) def test_dict_keys_time(self): - self.assertEqual( + assert ( orjson.dumps( {datetime.time(12, 15, 59, 111): True}, option=orjson.OPT_NON_STR_KEYS, - ), - b'{"12:15:59.000111":true}', + ) + == b'{"12:15:59.000111":true}' ) def test_dict_non_str_and_sort_keys(self): - self.assertEqual( + assert ( orjson.dumps( { "other": 1, @@ -227,8 +225,8 @@ def test_dict_non_str_and_sort_keys(self): datetime.date(1970, 1, 3): 3, }, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_SORT_KEYS, - ), - b'{"1970-01-03":3,"1970-01-05":2,"other":1}', + ) + == b'{"1970-01-03":3,"1970-01-05":2,"other":1}' ) @pytest.mark.skipif(pytz is None, reason="pytz optional") @@ -237,13 +235,12 @@ def test_dict_keys_time_err(self): OPT_NON_STR_KEYS propagates errors in types """ val = datetime.time(12, 15, 59, 111, tzinfo=pytz.timezone("Asia/Shanghai")) - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps({val: True}, option=orjson.OPT_NON_STR_KEYS) def test_dict_keys_str(self): - self.assertEqual( - orjson.dumps({"1": True}, option=orjson.OPT_NON_STR_KEYS), - b'{"1":true}', + assert ( + orjson.dumps({"1": True}, option=orjson.OPT_NON_STR_KEYS) == b'{"1":true}' ) def test_dict_keys_type(self): @@ -251,21 +248,21 @@ class Obj: a: str val = Obj() - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps({val: True}, option=orjson.OPT_NON_STR_KEYS) @pytest.mark.skipif(numpy is None, reason="numpy is not installed") def test_dict_keys_array(self): - with self.assertRaises(TypeError): - {numpy.array([1, 2]): True} + with pytest.raises(TypeError): + _ = {numpy.array([1, 2]): True} # type: ignore def test_dict_keys_dataclass(self): @dataclasses.dataclass class Dataclass: a: str - with self.assertRaises(TypeError): - {Dataclass("a"): True} + with pytest.raises(TypeError): + _ = {Dataclass("a"): True} def test_dict_keys_dataclass_hash(self): @dataclasses.dataclass @@ -276,25 +273,24 @@ def __hash__(self): return 1 obj = {Dataclass("a"): True} - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(obj, option=orjson.OPT_NON_STR_KEYS) def test_dict_keys_list(self): - with self.assertRaises(TypeError): - {[]: True} + with pytest.raises(TypeError): + _ = {[]: True} def test_dict_keys_dict(self): - with self.assertRaises(TypeError): - {{}: True} + with pytest.raises(TypeError): + _ = {{}: True} def test_dict_keys_tuple(self): obj = {(): True} - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(obj, option=orjson.OPT_NON_STR_KEYS) def test_dict_keys_unknown(self): - obj = {frozenset(): True} - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps({frozenset(): True}, option=orjson.OPT_NON_STR_KEYS) def test_dict_keys_no_str_call(self): @@ -305,5 +301,5 @@ def __str__(self): return "Obj" val = Obj() - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps({val: True}, option=orjson.OPT_NON_STR_KEYS) diff --git a/test/test_numpy.py b/test/test_numpy.py index b8f80da8..4f5339cf 100644 --- a/test/test_numpy.py +++ b/test/test_numpy.py @@ -1,124 +1,243 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2020-2026), Ben Sully (2021), Nazar Kostetskyi (2022), Aviram Hassan (2020-2021), Marco Ribeiro (2020), Eric Jolibois (2021) +# mypy: ignore-errors -import unittest +import sys import pytest import orjson -try: - import numpy -except ImportError: - numpy = None # type: ignore +from .util import numpy def numpy_default(obj): - return obj.tolist() + if isinstance(obj, numpy.ndarray): + return obj.tolist() + raise TypeError @pytest.mark.skipif(numpy is None, reason="numpy is not installed") -class NumpyTests(unittest.TestCase): +class TestNumpy: def test_numpy_array_d1_uintp(self): - self.assertEqual( - orjson.dumps( - numpy.array([0, 18446744073709551615], numpy.uintp), - option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[0,18446744073709551615]", - ) + low = numpy.iinfo(numpy.uintp).min + high = numpy.iinfo(numpy.uintp).max + assert orjson.dumps( + numpy.array([low, high], numpy.uintp), + option=orjson.OPT_SERIALIZE_NUMPY, + ) == f"[{low},{high}]".encode("ascii") def test_numpy_array_d1_intp(self): - self.assertEqual( - orjson.dumps( - numpy.array([-9223372036854775807, 9223372036854775807], numpy.intp), - option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[-9223372036854775807,9223372036854775807]", - ) + low = numpy.iinfo(numpy.intp).min + high = numpy.iinfo(numpy.intp).max + assert orjson.dumps( + numpy.array([low, high], numpy.intp), + option=orjson.OPT_SERIALIZE_NUMPY, + ) == f"[{low},{high}]".encode("ascii") def test_numpy_array_d1_i64(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array([-9223372036854775807, 9223372036854775807], numpy.int64), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[-9223372036854775807,9223372036854775807]", + ) + == b"[-9223372036854775807,9223372036854775807]" ) def test_numpy_array_d1_u64(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array([0, 18446744073709551615], numpy.uint64), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[0,18446744073709551615]", + ) + == b"[0,18446744073709551615]" ) def test_numpy_array_d1_i8(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array([-128, 127], numpy.int8), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[-128,127]", + ) + == b"[-128,127]" ) def test_numpy_array_d1_u8(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array([0, 255], numpy.uint8), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[0,255]", + ) + == b"[0,255]" ) def test_numpy_array_d1_i32(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array([-2147483647, 2147483647], numpy.int32), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[-2147483647,2147483647]", + ) + == b"[-2147483647,2147483647]" + ) + + def test_numpy_array_d1_i16(self): + assert ( + orjson.dumps( + numpy.array([-32768, 32767], numpy.int16), + option=orjson.OPT_SERIALIZE_NUMPY, + ) + == b"[-32768,32767]" + ) + + def test_numpy_array_d1_u16(self): + assert ( + orjson.dumps( + numpy.array([0, 65535], numpy.uint16), + option=orjson.OPT_SERIALIZE_NUMPY, + ) + == b"[0,65535]" ) def test_numpy_array_d1_u32(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array([0, 4294967295], numpy.uint32), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[0,4294967295]", + ) + == b"[0,4294967295]" ) def test_numpy_array_d1_f32(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array([1.0, 3.4028235e38], numpy.float32), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[1.0,3.4028235e38]", + ) + == b"[1.0,3.4028235e+38]" + ) + + def test_numpy_array_d1_f16(self): + assert ( + orjson.dumps( + numpy.array([-1.0, 0.0009765625, 1.0, 65504.0], numpy.float16), + option=orjson.OPT_SERIALIZE_NUMPY, + ) + == b"[-1.0,0.0009765625,1.0,65504.0]" + ) + + def test_numpy_array_f16_roundtrip(self): + ref = [ + -1.0, + -2.0, + 0.000000059604645, + 0.000060975552, + 0.00006103515625, + 0.0009765625, + 0.33325195, + 0.99951172, + 1.0, + 1.00097656, + 65504.0, + ] + obj = numpy.array(ref, numpy.float16) # type: ignore + serialized = orjson.dumps( + obj, + option=orjson.OPT_SERIALIZE_NUMPY, + ) + deserialized = numpy.array(orjson.loads(serialized), numpy.float16) # type: ignore + assert numpy.array_equal(obj, deserialized) + + def test_numpy_array_f16_edge(self): + assert ( + orjson.dumps( + numpy.array( + [ + numpy.inf, + -numpy.inf, + numpy.nan, + -0.0, + 0.0, + numpy.pi, + ], + numpy.float16, + ), + option=orjson.OPT_SERIALIZE_NUMPY, + ) + == b"[null,null,null,-0.0,0.0,3.140625]" + ) + + def test_numpy_array_f32_edge(self): + assert ( + orjson.dumps( + numpy.array( + [ + numpy.inf, + -numpy.inf, + numpy.nan, + -0.0, + 0.0, + numpy.pi, + ], + numpy.float32, + ), + option=orjson.OPT_SERIALIZE_NUMPY, + ) + == b"[null,null,null,-0.0,0.0,3.1415927]" + ) + + def test_numpy_array_f64_edge(self): + assert ( + orjson.dumps( + numpy.array( + [ + numpy.inf, + -numpy.inf, + numpy.nan, + -0.0, + 0.0, + numpy.pi, + ], + numpy.float64, + ), + option=orjson.OPT_SERIALIZE_NUMPY, + ) + == b"[null,null,null,-0.0,0.0,3.141592653589793]" ) def test_numpy_array_d1_f64(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array([1.0, 1.7976931348623157e308], numpy.float64), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[1.0,1.7976931348623157e308]", + ) + == b"[1.0,1.7976931348623157e+308]" ) def test_numpy_array_d1_bool(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array([True, False, False, True]), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[true,false,false,true]", + ) + == b"[true,false,false,true]" ) + def test_numpy_array_datetime_min_invalid(self): + """ + numpy.datetime64 min range invalid + """ + with pytest.raises(orjson.JSONEncodeError): + orjson.dumps(numpy.datetime64("-1"), option=orjson.OPT_SERIALIZE_NUMPY) + + def test_numpy_array_datetime_max_invalid(self): + """ + numpy.datetime64 max range invalid + """ + with pytest.raises(orjson.JSONEncodeError): + orjson.dumps(numpy.datetime64("10000"), option=orjson.OPT_SERIALIZE_NUMPY) + def test_numpy_array_d1_datetime64_years(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array( [ @@ -130,131 +249,131 @@ def test_numpy_array_d1_datetime64_years(self): numpy.datetime64("2022"), numpy.datetime64("2023"), numpy.datetime64("9999"), - ] + ], ), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b'["0001-01-01T00:00:00","0970-01-01T00:00:00","1920-01-01T00:00:00","1971-01-01T00:00:00","2021-01-01T00:00:00","2022-01-01T00:00:00","2023-01-01T00:00:00","9999-01-01T00:00:00"]', + ) + == b'["0001-01-01T00:00:00","0970-01-01T00:00:00","1920-01-01T00:00:00","1971-01-01T00:00:00","2021-01-01T00:00:00","2022-01-01T00:00:00","2023-01-01T00:00:00","9999-01-01T00:00:00"]' ) def test_numpy_array_d1_datetime64_months(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array( [ numpy.datetime64("2021-01"), numpy.datetime64("2022-01"), numpy.datetime64("2023-01"), - ] + ], ), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b'["2021-01-01T00:00:00","2022-01-01T00:00:00","2023-01-01T00:00:00"]', + ) + == b'["2021-01-01T00:00:00","2022-01-01T00:00:00","2023-01-01T00:00:00"]' ) def test_numpy_array_d1_datetime64_days(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array( [ numpy.datetime64("2021-01-01"), numpy.datetime64("2021-01-01"), numpy.datetime64("2021-01-01"), - ] + ], ), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b'["2021-01-01T00:00:00","2021-01-01T00:00:00","2021-01-01T00:00:00"]', + ) + == b'["2021-01-01T00:00:00","2021-01-01T00:00:00","2021-01-01T00:00:00"]' ) def test_numpy_array_d1_datetime64_hours(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array( [ numpy.datetime64("2021-01-01T00"), numpy.datetime64("2021-01-01T01"), numpy.datetime64("2021-01-01T02"), - ] + ], ), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b'["2021-01-01T00:00:00","2021-01-01T01:00:00","2021-01-01T02:00:00"]', + ) + == b'["2021-01-01T00:00:00","2021-01-01T01:00:00","2021-01-01T02:00:00"]' ) def test_numpy_array_d1_datetime64_minutes(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array( [ numpy.datetime64("2021-01-01T00:00"), numpy.datetime64("2021-01-01T00:01"), numpy.datetime64("2021-01-01T00:02"), - ] + ], ), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b'["2021-01-01T00:00:00","2021-01-01T00:01:00","2021-01-01T00:02:00"]', + ) + == b'["2021-01-01T00:00:00","2021-01-01T00:01:00","2021-01-01T00:02:00"]' ) def test_numpy_array_d1_datetime64_seconds(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array( [ numpy.datetime64("2021-01-01T00:00:00"), numpy.datetime64("2021-01-01T00:00:01"), numpy.datetime64("2021-01-01T00:00:02"), - ] + ], ), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b'["2021-01-01T00:00:00","2021-01-01T00:00:01","2021-01-01T00:00:02"]', + ) + == b'["2021-01-01T00:00:00","2021-01-01T00:00:01","2021-01-01T00:00:02"]' ) def test_numpy_array_d1_datetime64_milliseconds(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array( [ numpy.datetime64("2021-01-01T00:00:00"), numpy.datetime64("2021-01-01T00:00:00.172"), numpy.datetime64("2021-01-01T00:00:00.567"), - ] + ], ), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b'["2021-01-01T00:00:00","2021-01-01T00:00:00.172000","2021-01-01T00:00:00.567000"]', + ) + == b'["2021-01-01T00:00:00","2021-01-01T00:00:00.172000","2021-01-01T00:00:00.567000"]' ) def test_numpy_array_d1_datetime64_microseconds(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array( [ numpy.datetime64("2021-01-01T00:00:00"), numpy.datetime64("2021-01-01T00:00:00.172"), numpy.datetime64("2021-01-01T00:00:00.567891"), - ] + ], ), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b'["2021-01-01T00:00:00","2021-01-01T00:00:00.172000","2021-01-01T00:00:00.567891"]', + ) + == b'["2021-01-01T00:00:00","2021-01-01T00:00:00.172000","2021-01-01T00:00:00.567891"]' ) def test_numpy_array_d1_datetime64_nanoseconds(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array( [ numpy.datetime64("2021-01-01T00:00:00"), numpy.datetime64("2021-01-01T00:00:00.172"), numpy.datetime64("2021-01-01T00:00:00.567891234"), - ] + ], ), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b'["2021-01-01T00:00:00","2021-01-01T00:00:00.172000","2021-01-01T00:00:00.567891"]', + ) + == b'["2021-01-01T00:00:00","2021-01-01T00:00:00.172000","2021-01-01T00:00:00.567891"]' ) def test_numpy_array_d1_datetime64_picoseconds(self): @@ -265,215 +384,206 @@ def test_numpy_array_d1_datetime64_picoseconds(self): numpy.datetime64("2021-01-01T00:00:00"), numpy.datetime64("2021-01-01T00:00:00.172"), numpy.datetime64("2021-01-01T00:00:00.567891234567"), - ] + ], ), option=orjson.OPT_SERIALIZE_NUMPY, ) - assert False + raise AssertionError() except TypeError as exc: - self.assertEqual( - str(exc), - "unsupported numpy.datetime64 unit: picoseconds", - ) + assert str(exc) == "unsupported numpy.datetime64 unit: picoseconds" def test_numpy_array_d2_i64(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array([[1, 2, 3], [4, 5, 6]], numpy.int64), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[[1,2,3],[4,5,6]]", + ) + == b"[[1,2,3],[4,5,6]]" ) def test_numpy_array_d2_f64(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], numpy.float64), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[[1.0,2.0,3.0],[4.0,5.0,6.0]]", + ) + == b"[[1.0,2.0,3.0],[4.0,5.0,6.0]]" ) def test_numpy_array_d3_i8(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]], numpy.int8), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[[[1,2],[3,4]],[[5,6],[7,8]]]", + ) + == b"[[[1,2],[3,4]],[[5,6],[7,8]]]" ) def test_numpy_array_d3_u8(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]], numpy.uint8), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[[[1,2],[3,4]],[[5,6],[7,8]]]", + ) + == b"[[[1,2],[3,4]],[[5,6],[7,8]]]" ) def test_numpy_array_d3_i32(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]], numpy.int32), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[[[1,2],[3,4]],[[5,6],[7,8]]]", + ) + == b"[[[1,2],[3,4]],[[5,6],[7,8]]]" ) def test_numpy_array_d3_i64(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array([[[1, 2], [3, 4], [5, 6], [7, 8]]], numpy.int64), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[[[1,2],[3,4],[5,6],[7,8]]]", + ) + == b"[[[1,2],[3,4],[5,6],[7,8]]]" ) def test_numpy_array_d3_f64(self): - self.assertEqual( + assert ( orjson.dumps( numpy.array( - [[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]], numpy.float64 + [[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]], + numpy.float64, ), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[[[1.0,2.0],[3.0,4.0]],[[5.0,6.0],[7.0,8.0]]]", + ) + == b"[[[1.0,2.0],[3.0,4.0]],[[5.0,6.0],[7.0,8.0]]]" ) def test_numpy_array_fortran(self): array = numpy.array([[1, 2], [3, 4]], order="F") - assert array.flags["F_CONTIGUOUS"] == True - with self.assertRaises(orjson.JSONEncodeError): + assert array.flags["F_CONTIGUOUS"] is True + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(array, option=orjson.OPT_SERIALIZE_NUMPY) - self.assertEqual( - orjson.dumps( - array, default=numpy_default, option=orjson.OPT_SERIALIZE_NUMPY - ), - orjson.dumps(array.tolist()), - ) + assert orjson.dumps( + array, + default=numpy_default, + option=orjson.OPT_SERIALIZE_NUMPY, + ) == orjson.dumps(array.tolist()) def test_numpy_array_non_contiguous_message(self): array = numpy.array([[1, 2], [3, 4]], order="F") - assert array.flags["F_CONTIGUOUS"] == True + assert array.flags["F_CONTIGUOUS"] is True try: orjson.dumps(array, option=orjson.OPT_SERIALIZE_NUMPY) - assert False + raise AssertionError() except TypeError as exc: - self.assertEqual( - str(exc), - "numpy array is not C contiguous; use ndarray.tolist() in default", + assert ( + str(exc) + == "numpy array is not C contiguous; use ndarray.tolist() in default" ) def test_numpy_array_unsupported_dtype(self): - array = numpy.array([[1, 2], [3, 4]], numpy.float16) - with self.assertRaises(orjson.JSONEncodeError) as cm: + array = numpy.array([[1, 2], [3, 4]], numpy.csingle) # type: ignore + with pytest.raises(orjson.JSONEncodeError) as cm: orjson.dumps(array, option=orjson.OPT_SERIALIZE_NUMPY) - assert str(cm.exception) == "unsupported datatype in numpy array" - self.assertEqual( - orjson.dumps( - array, default=numpy_default, option=orjson.OPT_SERIALIZE_NUMPY - ), - orjson.dumps(array.tolist()), - ) + assert "unsupported datatype in numpy array" in str(cm) def test_numpy_array_d1(self): array = numpy.array([1]) - self.assertEqual( + assert ( orjson.loads( orjson.dumps( array, option=orjson.OPT_SERIALIZE_NUMPY, - ) - ), - array.tolist(), + ), + ) + == array.tolist() ) def test_numpy_array_d2(self): array = numpy.array([[1]]) - self.assertEqual( + assert ( orjson.loads( orjson.dumps( array, option=orjson.OPT_SERIALIZE_NUMPY, - ) - ), - array.tolist(), + ), + ) + == array.tolist() ) def test_numpy_array_d3(self): array = numpy.array([[[1]]]) - self.assertEqual( + assert ( orjson.loads( orjson.dumps( array, option=orjson.OPT_SERIALIZE_NUMPY, - ) - ), - array.tolist(), + ), + ) + == array.tolist() ) def test_numpy_array_d4(self): array = numpy.array([[[[1]]]]) - self.assertEqual( + assert ( orjson.loads( orjson.dumps( array, option=orjson.OPT_SERIALIZE_NUMPY, - ) - ), - array.tolist(), + ), + ) + == array.tolist() ) def test_numpy_array_4_stride(self): array = numpy.random.rand(4, 4, 4, 4) - self.assertEqual( + assert ( orjson.loads( orjson.dumps( array, option=orjson.OPT_SERIALIZE_NUMPY, - ) - ), - array.tolist(), + ), + ) + == array.tolist() ) def test_numpy_array_dimension_zero(self): array = numpy.array(0) assert array.ndim == 0 - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(array, option=orjson.OPT_SERIALIZE_NUMPY) array = numpy.empty((0, 4, 2)) - self.assertEqual( + assert ( orjson.loads( orjson.dumps( array, option=orjson.OPT_SERIALIZE_NUMPY, - ) - ), - array.tolist(), + ), + ) + == array.tolist() ) array = numpy.empty((4, 0, 2)) - self.assertEqual( + assert ( orjson.loads( orjson.dumps( array, option=orjson.OPT_SERIALIZE_NUMPY, - ) - ), - array.tolist(), + ), + ) + == array.tolist() ) array = numpy.empty((2, 4, 0)) - self.assertEqual( + assert ( orjson.loads( orjson.dumps( array, option=orjson.OPT_SERIALIZE_NUMPY, - ) - ), - array.tolist(), + ), + ) + == array.tolist() ) def test_numpy_array_dimension_max(self): @@ -512,196 +622,533 @@ def test_numpy_array_dimension_max(self): 1, ) assert array.ndim == 32 - self.assertEqual( + assert ( orjson.loads( orjson.dumps( array, option=orjson.OPT_SERIALIZE_NUMPY, - ) - ), - array.tolist(), + ), + ) + == array.tolist() ) def test_numpy_scalar_int8(self): - self.assertEqual( - orjson.dumps(numpy.int8(0), option=orjson.OPT_SERIALIZE_NUMPY), b"0" + assert orjson.dumps(numpy.int8(0), option=orjson.OPT_SERIALIZE_NUMPY) == b"0" + assert ( + orjson.dumps(numpy.int8(127), option=orjson.OPT_SERIALIZE_NUMPY) == b"127" ) - self.assertEqual( - orjson.dumps(numpy.int8(127), option=orjson.OPT_SERIALIZE_NUMPY), - b"127", + assert ( + orjson.dumps(numpy.int8(-128), option=orjson.OPT_SERIALIZE_NUMPY) == b"-128" ) - self.assertEqual( - orjson.dumps(numpy.int8(--128), option=orjson.OPT_SERIALIZE_NUMPY), - b"-128", + + def test_numpy_scalar_int16(self): + assert orjson.dumps(numpy.int16(0), option=orjson.OPT_SERIALIZE_NUMPY) == b"0" + assert ( + orjson.dumps(numpy.int16(32767), option=orjson.OPT_SERIALIZE_NUMPY) + == b"32767" + ) + assert ( + orjson.dumps(numpy.int16(-32768), option=orjson.OPT_SERIALIZE_NUMPY) + == b"-32768" ) def test_numpy_scalar_int32(self): - self.assertEqual( - orjson.dumps(numpy.int32(1), option=orjson.OPT_SERIALIZE_NUMPY), b"1" - ) - self.assertEqual( - orjson.dumps(numpy.int32(2147483647), option=orjson.OPT_SERIALIZE_NUMPY), - b"2147483647", + assert orjson.dumps(numpy.int32(1), option=orjson.OPT_SERIALIZE_NUMPY) == b"1" + assert ( + orjson.dumps(numpy.int32(2147483647), option=orjson.OPT_SERIALIZE_NUMPY) + == b"2147483647" ) - self.assertEqual( - orjson.dumps(numpy.int32(-2147483648), option=orjson.OPT_SERIALIZE_NUMPY), - b"-2147483648", + assert ( + orjson.dumps(numpy.int32(-2147483648), option=orjson.OPT_SERIALIZE_NUMPY) + == b"-2147483648" ) def test_numpy_scalar_int64(self): - self.assertEqual( + assert ( orjson.dumps( - numpy.int64(-9223372036854775808), option=orjson.OPT_SERIALIZE_NUMPY - ), - b"-9223372036854775808", + numpy.int64(-9223372036854775808), + option=orjson.OPT_SERIALIZE_NUMPY, + ) + == b"-9223372036854775808" ) - self.assertEqual( + assert ( orjson.dumps( - numpy.int64(9223372036854775807), option=orjson.OPT_SERIALIZE_NUMPY - ), - b"9223372036854775807", + numpy.int64(9223372036854775807), + option=orjson.OPT_SERIALIZE_NUMPY, + ) + == b"9223372036854775807" ) def test_numpy_scalar_uint8(self): - self.assertEqual( - orjson.dumps(numpy.uint8(0), option=orjson.OPT_SERIALIZE_NUMPY), b"0" + assert orjson.dumps(numpy.uint8(0), option=orjson.OPT_SERIALIZE_NUMPY) == b"0" + assert ( + orjson.dumps(numpy.uint8(255), option=orjson.OPT_SERIALIZE_NUMPY) == b"255" ) - self.assertEqual( - orjson.dumps(numpy.uint8(255), option=orjson.OPT_SERIALIZE_NUMPY), - b"255", + + def test_numpy_scalar_uint16(self): + assert orjson.dumps(numpy.uint16(0), option=orjson.OPT_SERIALIZE_NUMPY) == b"0" + assert ( + orjson.dumps(numpy.uint16(65535), option=orjson.OPT_SERIALIZE_NUMPY) + == b"65535" ) def test_numpy_scalar_uint32(self): - self.assertEqual( - orjson.dumps(numpy.uint32(0), option=orjson.OPT_SERIALIZE_NUMPY), b"0" - ) - self.assertEqual( - orjson.dumps(numpy.uint32(4294967295), option=orjson.OPT_SERIALIZE_NUMPY), - b"4294967295", + assert orjson.dumps(numpy.uint32(0), option=orjson.OPT_SERIALIZE_NUMPY) == b"0" + assert ( + orjson.dumps(numpy.uint32(4294967295), option=orjson.OPT_SERIALIZE_NUMPY) + == b"4294967295" ) def test_numpy_scalar_uint64(self): - self.assertEqual( - orjson.dumps(numpy.uint64(0), option=orjson.OPT_SERIALIZE_NUMPY), b"0" - ) - self.assertEqual( + assert orjson.dumps(numpy.uint64(0), option=orjson.OPT_SERIALIZE_NUMPY) == b"0" + assert ( orjson.dumps( - numpy.uint64(18446744073709551615), option=orjson.OPT_SERIALIZE_NUMPY - ), - b"18446744073709551615", + numpy.uint64(18446744073709551615), + option=orjson.OPT_SERIALIZE_NUMPY, + ) + == b"18446744073709551615" + ) + + def test_numpy_scalar_float16(self): + assert ( + orjson.dumps(numpy.float16(1.0), option=orjson.OPT_SERIALIZE_NUMPY) + == b"1.0" ) def test_numpy_scalar_float32(self): - self.assertEqual( - orjson.dumps(numpy.float32(1.0), option=orjson.OPT_SERIALIZE_NUMPY), b"1.0" + assert ( + orjson.dumps(numpy.float32(1.0), option=orjson.OPT_SERIALIZE_NUMPY) + == b"1.0" ) def test_numpy_scalar_float64(self): - self.assertEqual( - orjson.dumps(numpy.float64(123.123), option=orjson.OPT_SERIALIZE_NUMPY), - b"123.123", + assert ( + orjson.dumps(numpy.float64(123.123), option=orjson.OPT_SERIALIZE_NUMPY) + == b"123.123" ) def test_numpy_bool(self): - self.assertEqual( + assert ( orjson.dumps( {"a": numpy.bool_(True), "b": numpy.bool_(False)}, option=orjson.OPT_SERIALIZE_NUMPY, - ), - b'{"a":true,"b":false}', + ) + == b'{"a":true,"b":false}' + ) + + def test_numpy_datetime_year(self): + assert ( + orjson.dumps(numpy.datetime64("2021"), option=orjson.OPT_SERIALIZE_NUMPY) + == b'"2021-01-01T00:00:00"' ) - def test_numpy_datetime(self): - self.assertEqual( + def test_numpy_datetime_month(self): + assert ( + orjson.dumps(numpy.datetime64("2021-01"), option=orjson.OPT_SERIALIZE_NUMPY) + == b'"2021-01-01T00:00:00"' + ) + + def test_numpy_datetime_day(self): + assert ( orjson.dumps( - { - "year": numpy.datetime64("2021"), - "month": numpy.datetime64("2021-01"), - "day": numpy.datetime64("2021-01-01"), - "hour": numpy.datetime64("2021-01-01T00"), - "minute": numpy.datetime64("2021-01-01T00:00"), - "second": numpy.datetime64("2021-01-01T00:00:00"), - "milli": numpy.datetime64("2021-01-01T00:00:00.172"), - "micro": numpy.datetime64("2021-01-01T00:00:00.172576"), - "nano": numpy.datetime64("2021-01-01T00:00:00.172576789"), - }, + numpy.datetime64("2021-01-01"), option=orjson.OPT_SERIALIZE_NUMPY, - ), - b'{"year":"2021-01-01T00:00:00","month":"2021-01-01T00:00:00","day":"2021-01-01T00:00:00","hour":"2021-01-01T00:00:00","minute":"2021-01-01T00:00:00","second":"2021-01-01T00:00:00","milli":"2021-01-01T00:00:00.172000","micro":"2021-01-01T00:00:00.172576","nano":"2021-01-01T00:00:00.172576"}', - ) - - def test_numpy_datetime_naive_utc(self): - self.assertEqual( - orjson.dumps( - { - "year": numpy.datetime64("2021"), - "month": numpy.datetime64("2021-01"), - "day": numpy.datetime64("2021-01-01"), - "hour": numpy.datetime64("2021-01-01T00"), - "minute": numpy.datetime64("2021-01-01T00:00"), - "second": numpy.datetime64("2021-01-01T00:00:00"), - "milli": numpy.datetime64("2021-01-01T00:00:00.172"), - "micro": numpy.datetime64("2021-01-01T00:00:00.172576"), - "nano": numpy.datetime64("2021-01-01T00:00:00.172576789"), - }, + ) + == b'"2021-01-01T00:00:00"' + ) + + def test_numpy_datetime_hour(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00"), + option=orjson.OPT_SERIALIZE_NUMPY, + ) + == b'"2021-01-01T00:00:00"' + ) + + def test_numpy_datetime_minute(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00"), + option=orjson.OPT_SERIALIZE_NUMPY, + ) + == b'"2021-01-01T00:00:00"' + ) + + def test_numpy_datetime_second(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00:00"), + option=orjson.OPT_SERIALIZE_NUMPY, + ) + == b'"2021-01-01T00:00:00"' + ) + + def test_numpy_datetime_milli(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00:00.172"), + option=orjson.OPT_SERIALIZE_NUMPY, + ) + == b'"2021-01-01T00:00:00.172000"' + ) + + def test_numpy_datetime_micro(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00:00.172576"), + option=orjson.OPT_SERIALIZE_NUMPY, + ) + == b'"2021-01-01T00:00:00.172576"' + ) + + def test_numpy_datetime_nano(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00:00.172576789"), + option=orjson.OPT_SERIALIZE_NUMPY, + ) + == b'"2021-01-01T00:00:00.172576"' + ) + + def test_numpy_datetime_naive_utc_year(self): + assert ( + orjson.dumps( + numpy.datetime64("2021"), option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_NAIVE_UTC, - ), - b'{"year":"2021-01-01T00:00:00+00:00","month":"2021-01-01T00:00:00+00:00","day":"2021-01-01T00:00:00+00:00","hour":"2021-01-01T00:00:00+00:00","minute":"2021-01-01T00:00:00+00:00","second":"2021-01-01T00:00:00+00:00","milli":"2021-01-01T00:00:00.172000+00:00","micro":"2021-01-01T00:00:00.172576+00:00","nano":"2021-01-01T00:00:00.172576+00:00"}', - ) - - def test_numpy_datetime_naive_utc_utc_z(self): - self.assertEqual( - orjson.dumps( - { - "year": numpy.datetime64("2021"), - "month": numpy.datetime64("2021-01"), - "day": numpy.datetime64("2021-01-01"), - "hour": numpy.datetime64("2021-01-01T00"), - "minute": numpy.datetime64("2021-01-01T00:00"), - "second": numpy.datetime64("2021-01-01T00:00:00"), - "milli": numpy.datetime64("2021-01-01T00:00:00.172"), - "micro": numpy.datetime64("2021-01-01T00:00:00.172576"), - "nano": numpy.datetime64("2021-01-01T00:00:00.172576789"), - }, + ) + == b'"2021-01-01T00:00:00+00:00"' + ) + + def test_numpy_datetime_naive_utc_month(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01"), + option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_NAIVE_UTC, + ) + == b'"2021-01-01T00:00:00+00:00"' + ) + + def test_numpy_datetime_naive_utc_day(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01"), + option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_NAIVE_UTC, + ) + == b'"2021-01-01T00:00:00+00:00"' + ) + + def test_numpy_datetime_naive_utc_hour(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00"), + option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_NAIVE_UTC, + ) + == b'"2021-01-01T00:00:00+00:00"' + ) + + def test_numpy_datetime_naive_utc_minute(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00"), + option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_NAIVE_UTC, + ) + == b'"2021-01-01T00:00:00+00:00"' + ) + + def test_numpy_datetime_naive_utc_second(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00:00"), + option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_NAIVE_UTC, + ) + == b'"2021-01-01T00:00:00+00:00"' + ) + + def test_numpy_datetime_naive_utc_milli(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00:00.172"), + option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_NAIVE_UTC, + ) + == b'"2021-01-01T00:00:00.172000+00:00"' + ) + + def test_numpy_datetime_naive_utc_micro(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00:00.172576"), + option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_NAIVE_UTC, + ) + == b'"2021-01-01T00:00:00.172576+00:00"' + ) + + def test_numpy_datetime_naive_utc_nano(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00:00.172576789"), + option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_NAIVE_UTC, + ) + == b'"2021-01-01T00:00:00.172576+00:00"' + ) + + def test_numpy_datetime_naive_utc_utc_z_year(self): + assert ( + orjson.dumps( + numpy.datetime64("2021"), option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_NAIVE_UTC | orjson.OPT_UTC_Z, - ), - b'{"year":"2021-01-01T00:00:00Z","month":"2021-01-01T00:00:00Z","day":"2021-01-01T00:00:00Z","hour":"2021-01-01T00:00:00Z","minute":"2021-01-01T00:00:00Z","second":"2021-01-01T00:00:00Z","milli":"2021-01-01T00:00:00.172000Z","micro":"2021-01-01T00:00:00.172576Z","nano":"2021-01-01T00:00:00.172576Z"}', - ) - - def test_numpy_datetime_omit_microseconds(self): - self.assertEqual( - orjson.dumps( - { - "year": numpy.datetime64("2021"), - "month": numpy.datetime64("2021-01"), - "day": numpy.datetime64("2021-01-01"), - "hour": numpy.datetime64("2021-01-01T00"), - "minute": numpy.datetime64("2021-01-01T00:00"), - "second": numpy.datetime64("2021-01-01T00:00:00"), - "milli": numpy.datetime64("2021-01-01T00:00:00.172"), - "micro": numpy.datetime64("2021-01-01T00:00:00.172576"), - "nano": numpy.datetime64("2021-01-01T00:00:00.172576789"), - }, + ) + == b'"2021-01-01T00:00:00Z"' + ) + + def test_numpy_datetime_naive_utc_utc_z_month(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01"), + option=orjson.OPT_SERIALIZE_NUMPY + | orjson.OPT_NAIVE_UTC + | orjson.OPT_UTC_Z, + ) + == b'"2021-01-01T00:00:00Z"' + ) + + def test_numpy_datetime_naive_utc_utc_z_day(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01"), + option=orjson.OPT_SERIALIZE_NUMPY + | orjson.OPT_NAIVE_UTC + | orjson.OPT_UTC_Z, + ) + == b'"2021-01-01T00:00:00Z"' + ) + + def test_numpy_datetime_naive_utc_utc_z_hour(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00"), + option=orjson.OPT_SERIALIZE_NUMPY + | orjson.OPT_NAIVE_UTC + | orjson.OPT_UTC_Z, + ) + == b'"2021-01-01T00:00:00Z"' + ) + + def test_numpy_datetime_naive_utc_utc_z_minute(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00"), + option=orjson.OPT_SERIALIZE_NUMPY + | orjson.OPT_NAIVE_UTC + | orjson.OPT_UTC_Z, + ) + == b'"2021-01-01T00:00:00Z"' + ) + + def test_numpy_datetime_naive_utc_utc_z_second(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00:00"), + option=orjson.OPT_SERIALIZE_NUMPY + | orjson.OPT_NAIVE_UTC + | orjson.OPT_UTC_Z, + ) + == b'"2021-01-01T00:00:00Z"' + ) + + def test_numpy_datetime_naive_utc_utc_z_milli(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00:00.172"), + option=orjson.OPT_SERIALIZE_NUMPY + | orjson.OPT_NAIVE_UTC + | orjson.OPT_UTC_Z, + ) + == b'"2021-01-01T00:00:00.172000Z"' + ) + + def test_numpy_datetime_naive_utc_utc_z_micro(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00:00.172576"), + option=orjson.OPT_SERIALIZE_NUMPY + | orjson.OPT_NAIVE_UTC + | orjson.OPT_UTC_Z, + ) + == b'"2021-01-01T00:00:00.172576Z"' + ) + + def test_numpy_datetime_naive_utc_utc_z_nano(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00:00.172576789"), + option=orjson.OPT_SERIALIZE_NUMPY + | orjson.OPT_NAIVE_UTC + | orjson.OPT_UTC_Z, + ) + == b'"2021-01-01T00:00:00.172576Z"' + ) + + def test_numpy_datetime_omit_microseconds_year(self): + assert ( + orjson.dumps( + numpy.datetime64("2021"), option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_OMIT_MICROSECONDS, - ), - b'{"year":"2021-01-01T00:00:00","month":"2021-01-01T00:00:00","day":"2021-01-01T00:00:00","hour":"2021-01-01T00:00:00","minute":"2021-01-01T00:00:00","second":"2021-01-01T00:00:00","milli":"2021-01-01T00:00:00","micro":"2021-01-01T00:00:00","nano":"2021-01-01T00:00:00"}', + ) + == b'"2021-01-01T00:00:00"' + ) + + def test_numpy_datetime_omit_microseconds_month(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01"), + option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_OMIT_MICROSECONDS, + ) + == b'"2021-01-01T00:00:00"' + ) + + def test_numpy_datetime_omit_microseconds_day(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01"), + option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_OMIT_MICROSECONDS, + ) + == b'"2021-01-01T00:00:00"' + ) + + def test_numpy_datetime_omit_microseconds_hour(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00"), + option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_OMIT_MICROSECONDS, + ) + == b'"2021-01-01T00:00:00"' + ) + + def test_numpy_datetime_omit_microseconds_minute(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00"), + option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_OMIT_MICROSECONDS, + ) + == b'"2021-01-01T00:00:00"' + ) + + def test_numpy_datetime_omit_microseconds_second(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00:00"), + option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_OMIT_MICROSECONDS, + ) + == b'"2021-01-01T00:00:00"' + ) + + def test_numpy_datetime_omit_microseconds_milli(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00:00.172"), + option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_OMIT_MICROSECONDS, + ) + == b'"2021-01-01T00:00:00"' + ) + + def test_numpy_datetime_omit_microseconds_micro(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00:00.172576"), + option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_OMIT_MICROSECONDS, + ) + == b'"2021-01-01T00:00:00"' + ) + + def test_numpy_datetime_omit_microseconds_nano(self): + assert ( + orjson.dumps( + numpy.datetime64("2021-01-01T00:00:00.172576789"), + option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_OMIT_MICROSECONDS, + ) + == b'"2021-01-01T00:00:00"' ) def test_numpy_datetime_nat(self): - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(numpy.datetime64("NaT"), option=orjson.OPT_SERIALIZE_NUMPY) - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps([numpy.datetime64("NaT")], option=orjson.OPT_SERIALIZE_NUMPY) def test_numpy_repeated(self): - data = numpy.array([[[1, 2], [3, 4], [5, 6], [7, 8]]], numpy.int64) - for _ in range(0, 3): - self.assertEqual( + data = numpy.array([[[1, 2], [3, 4], [5, 6], [7, 8]]], numpy.int64) # type: ignore + for _ in range(3): + assert ( orjson.dumps( data, option=orjson.OPT_SERIALIZE_NUMPY, - ), - b"[[[1,2],[3,4],[5,6],[7,8]]]", + ) + == b"[[[1,2],[3,4],[5,6],[7,8]]]" ) + + +@pytest.mark.skipif(numpy is None, reason="numpy is not installed") +class TestNumpyEquivalence: + def _test(self, obj): + assert orjson.dumps(obj, option=orjson.OPT_SERIALIZE_NUMPY) == orjson.dumps( + obj.tolist(), + ) + + def test_numpy_uint8(self): + self._test(numpy.array([0, 255], numpy.uint8)) + + def test_numpy_uint16(self): + self._test(numpy.array([0, 65535], numpy.uint16)) + + def test_numpy_uint32(self): + self._test(numpy.array([0, 4294967295], numpy.uint32)) + + def test_numpy_uint64(self): + self._test(numpy.array([0, 18446744073709551615], numpy.uint64)) + + def test_numpy_int8(self): + self._test(numpy.array([-128, 127], numpy.int8)) + + def test_numpy_int16(self): + self._test(numpy.array([-32768, 32767], numpy.int16)) + + def test_numpy_int32(self): + self._test(numpy.array([-2147483647, 2147483647], numpy.int32)) + + def test_numpy_int64(self): + self._test( + numpy.array([-9223372036854775807, 9223372036854775807], numpy.int64), + ) + + @pytest.mark.skip(reason="tolist() conversion results in 3.4028234663852886e38") + def test_numpy_float32(self): + self._test( + numpy.array( + [ + -340282346638528859811704183484516925440.0000000000000000, + 340282346638528859811704183484516925440.0000000000000000, + ], + numpy.float32, + ), + ) + self._test(numpy.array([-3.4028235e38, 3.4028235e38], numpy.float32)) + + def test_numpy_float64(self): + self._test( + numpy.array( + [-1.7976931348623157e308, 1.7976931348623157e308], + numpy.float64, + ), + ) + + +@pytest.mark.skipif(numpy is None, reason="numpy is not installed") +class NumpyEndianness: + def test_numpy_array_dimension_zero(self): + wrong_endianness = ">" if sys.byteorder == "little" else "<" + array = numpy.array([0, 1, 0.4, 5.7], dtype=f"{wrong_endianness}f8") + with pytest.raises(orjson.JSONEncodeError): + orjson.dumps(array, option=orjson.OPT_SERIALIZE_NUMPY) diff --git a/test/test_parsing.py b/test/test_parsing.py index d3bac9f5..a2b36bec 100644 --- a/test/test_parsing.py +++ b/test/test_parsing.py @@ -1,34 +1,45 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2018-2026) -import unittest +import pytest import orjson -from .util import read_fixture_bytes +from .util import ( + SUPPORTS_BYTEARRAY, + SUPPORTS_MEMORYVIEW, + needs_data, + read_fixture_bytes, +) -class JSONTestSuiteParsingTests(unittest.TestCase): +@needs_data +class TestJSONTestSuiteParsing: def _run_fail_json(self, filename, exc=orjson.JSONDecodeError): data = read_fixture_bytes(filename, "parsing") - with self.assertRaises(exc, msg=data): - res = orjson.loads(data) - with self.assertRaises(exc, msg=data): - res = orjson.loads(bytearray(data)) - with self.assertRaises(exc, msg=data): - res = orjson.loads(memoryview(data)) + with pytest.raises(exc): + orjson.loads(data) + if SUPPORTS_BYTEARRAY: + with pytest.raises(exc): + orjson.loads(bytearray(data)) + if SUPPORTS_MEMORYVIEW: + with pytest.raises(exc): + orjson.loads(memoryview(data)) try: decoded = data.decode("utf-8") except UnicodeDecodeError: pass else: - with self.assertRaises(exc, msg=decoded): - res = orjson.loads(decoded) + with pytest.raises(exc): + orjson.loads(decoded) def _run_pass_json(self, filename, match=""): data = read_fixture_bytes(filename, "parsing") orjson.loads(data) - orjson.loads(bytearray(data)) - orjson.loads(memoryview(data)) + if SUPPORTS_BYTEARRAY: + orjson.loads(bytearray(data)) + if SUPPORTS_MEMORYVIEW: + orjson.loads(memoryview(data)) orjson.loads(data.decode("utf-8")) def test_y_array_arraysWithSpace(self): @@ -1134,7 +1145,7 @@ def test_n_object_lone_continuation_byte_in_key_and_trailing_comma(self): n_object_lone_continuation_byte_in_key_and_trailing_comma.json """ self._run_fail_json( - "n_object_lone_continuation_byte_in_key_and_trailing_comma.json" + "n_object_lone_continuation_byte_in_key_and_trailing_comma.json", ) def test_n_object_missing_col(self): @@ -1933,11 +1944,7 @@ def test_i_structure_500_nested_array(self): """ i_structure_500_nested_arrays.json """ - try: - self._run_pass_json("i_structure_500_nested_arrays.json.xz") - except orjson.JSONDecodeError: - # fails on serde, passes on yyjson - pass + self._run_pass_json("i_structure_500_nested_arrays.json.xz") def test_i_structure_UTF_8_BOM_empty_object(self): """ diff --git a/test/test_recursion.py b/test/test_recursion.py new file mode 100644 index 00000000..74f6bf06 --- /dev/null +++ b/test/test_recursion.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2026) + + +import pytest + +import orjson + + +def make_recursive_list_dict(limit: int, envelope_key: str, recurse_key: str): + i = 0 + root = [{envelope_key: i, recurse_key: []}] + i += 1 + while i < limit: + sub = [{envelope_key: i, recurse_key: []}] + sub[0][recurse_key] = root + root = sub + i += 1 + return root + + +class TestSerializeRecursion: + @pytest.mark.parametrize("i", range(1, 127)) + def test_dumps_recursion_valid_long(self, i): + root = make_recursive_list_dict(i, "🐈" * 512, "b" * 1024) + orjson.dumps(root) + + @pytest.mark.parametrize("i", range(1, 127)) + def test_dumps_recursion_valid_short_1(self, i): + root = make_recursive_list_dict(i, "a", "") + orjson.dumps(root) + + @pytest.mark.parametrize("i", range(1, 127)) + def test_dumps_recursion_valid_short_2(self, i): + root = make_recursive_list_dict(i, "level", "next") + orjson.dumps(root) + + def test_dumps_recursion_limit(self): + root = make_recursive_list_dict(128, "level", "next") + with pytest.raises(orjson.JSONEncodeError): + orjson.dumps(root) diff --git a/test/test_reentrant.py b/test/test_reentrant.py new file mode 100644 index 00000000..42bea6b4 --- /dev/null +++ b/test/test_reentrant.py @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright Anders Kaseorg (2023) + +import orjson + + +class C: + c: "C" + + def __del__(self): + orjson.loads('"' + "a" * 10000 + '"') + + +def test_reentrant(): + c = C() + c.c = c + del c + + orjson.loads("[" + "[]," * 1000 + "[]]") diff --git a/test/test_roundtrip.py b/test/test_roundtrip.py index 4fd846cf..69c2e95a 100644 --- a/test/test_roundtrip.py +++ b/test/test_roundtrip.py @@ -1,16 +1,19 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import unittest +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2018-2026) import orjson -from .util import read_fixture_str +from .util import needs_data, read_fixture_str -class JsonCheckerTests(unittest.TestCase): - def _run_roundtrip_json(self, filename): +@needs_data +class TestJsonChecker: + def _run_roundtrip_json(self, filename, byte_exact=True): data = read_fixture_str(filename, "roundtrip") - self.assertEqual(orjson.dumps(orjson.loads(data)), data.encode("utf-8")) + if byte_exact: + assert orjson.dumps(orjson.loads(data)) == data.encode("utf-8") + else: + assert orjson.loads(orjson.dumps(orjson.loads(data))) == orjson.loads(data) def test_roundtrip001(self): """ @@ -172,4 +175,4 @@ def test_roundtrip027(self): """ roundtrip027.json """ - self._run_roundtrip_json("roundtrip27.json") + self._run_roundtrip_json("roundtrip27.json", byte_exact=False) diff --git a/test/test_sort_keys.py b/test/test_sort_keys.py index 3b55e437..8257308e 100644 --- a/test/test_sort_keys.py +++ b/test/test_sort_keys.py @@ -1,33 +1,33 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import unittest +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2020-2025) import orjson -from .util import read_fixture_obj +from .util import needs_data, read_fixture_obj -class DictSortKeysTests(unittest.TestCase): +@needs_data +class TestDictSortKeys: # citm_catalog is already sorted def test_twitter_sorted(self): """ twitter.json sorted """ obj = read_fixture_obj("twitter.json.xz") - self.assertNotEqual(list(obj.keys()), sorted(list(obj.keys()))) + assert list(obj.keys()) != sorted(list(obj.keys())) serialized = orjson.dumps(obj, option=orjson.OPT_SORT_KEYS) val = orjson.loads(serialized) - self.assertEqual(list(val.keys()), sorted(list(val.keys()))) + assert list(val.keys()) == sorted(list(val.keys())) def test_canada_sorted(self): """ canada.json sorted """ obj = read_fixture_obj("canada.json.xz") - self.assertNotEqual(list(obj.keys()), sorted(list(obj.keys()))) + assert list(obj.keys()) != sorted(list(obj.keys())) serialized = orjson.dumps(obj, option=orjson.OPT_SORT_KEYS) val = orjson.loads(serialized) - self.assertEqual(list(val.keys()), sorted(list(val.keys()))) + assert list(val.keys()) == sorted(list(val.keys())) def test_github_sorted(self): """ @@ -35,18 +35,18 @@ def test_github_sorted(self): """ obj = read_fixture_obj("github.json.xz") for each in obj: - self.assertNotEqual(list(each.keys()), sorted(list(each.keys()))) + assert list(each.keys()) != sorted(list(each.keys())) serialized = orjson.dumps(obj, option=orjson.OPT_SORT_KEYS) val = orjson.loads(serialized) for each in val: - self.assertEqual(list(each.keys()), sorted(list(each.keys()))) + assert list(each.keys()) == sorted(list(each.keys())) def test_utf8_sorted(self): """ UTF-8 sorted """ obj = {"a": 1, "ä": 2, "A": 3} - self.assertNotEqual(list(obj.keys()), sorted(list(obj.keys()))) + assert list(obj.keys()) != sorted(list(obj.keys())) serialized = orjson.dumps(obj, option=orjson.OPT_SORT_KEYS) val = orjson.loads(serialized) - self.assertEqual(list(val.keys()), sorted(list(val.keys()))) + assert list(val.keys()) == sorted(list(val.keys())) diff --git a/test/test_subclass.py b/test/test_subclass.py index 1bb66a75..bb8c2036 100644 --- a/test/test_subclass.py +++ b/test/test_subclass.py @@ -1,8 +1,10 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2018-2022) import collections import json -import unittest + +import pytest import orjson @@ -31,96 +33,81 @@ class SubTuple(tuple): pass -class SubclassTests(unittest.TestCase): +class TestSubclass: def test_subclass_str(self): - self.assertEqual( - orjson.dumps(SubStr("zxc")), - b'"zxc"', - ) + assert orjson.dumps(SubStr("zxc")) == b'"zxc"' def test_subclass_str_invalid(self): - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(SubStr("\ud800")) def test_subclass_int(self): - self.assertEqual(orjson.dumps(SubInt(1)), b"1") + assert orjson.dumps(SubInt(1)) == b"1" def test_subclass_int_64(self): for val in (9223372036854775807, -9223372036854775807): - self.assertEqual(orjson.dumps(SubInt(val)), str(val).encode("utf-8")) + assert orjson.dumps(SubInt(val)) == str(val).encode("utf-8") def test_subclass_int_53(self): for val in (9007199254740992, -9007199254740992): - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(SubInt(val), option=orjson.OPT_STRICT_INTEGER) def test_subclass_dict(self): - self.assertEqual( - orjson.dumps(SubDict({"a": "b"})), - b'{"a":"b"}', - ) + assert orjson.dumps(SubDict({"a": "b"})) == b'{"a":"b"}' def test_subclass_list(self): - self.assertEqual( - orjson.dumps(SubList(["a", "b"])), - b'["a","b"]', - ) + assert orjson.dumps(SubList(["a", "b"])) == b'["a","b"]' ref = [True] * 512 - self.assertEqual(orjson.loads(orjson.dumps(SubList(ref))), ref) + assert orjson.loads(orjson.dumps(SubList(ref))) == ref def test_subclass_float(self): - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(SubFloat(1.1)) - self.assertEqual( - json.dumps(SubFloat(1.1)), - "1.1", - ) + assert json.dumps(SubFloat(1.1)) == "1.1" def test_subclass_tuple(self): - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(SubTuple((1, 2))) - self.assertEqual( - json.dumps(SubTuple((1, 2))), - "[1, 2]", - ) + assert json.dumps(SubTuple((1, 2))) == "[1, 2]" def test_namedtuple(self): Point = collections.namedtuple("Point", ["x", "y"]) - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(Point(1, 2)) def test_subclass_circular_dict(self): obj = SubDict({}) obj["obj"] = obj - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(obj) def test_subclass_circular_list(self): obj = SubList([]) obj.append(obj) - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(obj) def test_subclass_circular_nested(self): obj = SubDict({}) obj["list"] = SubList([{"obj": obj}]) - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(obj) -class SubclassPassthroughTests(unittest.TestCase): +class TestSubclassPassthrough: def test_subclass_str(self): - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(SubStr("zxc"), option=orjson.OPT_PASSTHROUGH_SUBCLASS) def test_subclass_int(self): - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(SubInt(1), option=orjson.OPT_PASSTHROUGH_SUBCLASS) def test_subclass_dict(self): - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(SubDict({"a": "b"}), option=orjson.OPT_PASSTHROUGH_SUBCLASS) def test_subclass_list(self): - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(SubList(["a", "b"]), option=orjson.OPT_PASSTHROUGH_SUBCLASS) diff --git a/test/test_transform.py b/test/test_transform.py index 25ec3fd4..1acc088b 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -1,24 +1,26 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2019-2025) -import unittest +import pytest import orjson -from .util import read_fixture_bytes +from .util import needs_data, read_fixture_bytes def _read_file(filename): return read_fixture_bytes(filename, "transform").strip(b"\n").strip(b"\r") -class JSONTestSuiteTransformTests(unittest.TestCase): +@needs_data +class TestJSONTestSuiteTransform: def _pass_transform(self, filename, reference=None): data = _read_file(filename) - self.assertEqual(orjson.dumps(orjson.loads(data)), reference or data) + assert orjson.dumps(orjson.loads(data)) == (reference or data) def _fail_transform(self, filename): data = _read_file(filename) - with self.assertRaises(orjson.JSONDecodeError): + with pytest.raises(orjson.JSONDecodeError): orjson.loads(data) def test_number_1(self): @@ -44,10 +46,9 @@ def test_number_10000000000000000999(self): number_10000000000000000999.json """ # cannot serialize due to range - self.assertEqual( - orjson.loads(_read_file("number_10000000000000000999.json")), - [10000000000000000999], - ) + assert orjson.loads(_read_file("number_10000000000000000999.json")) == [ + 10000000000000000999, + ] def test_number_1000000000000000(self): """ diff --git a/test/test_type.py b/test/test_type.py index 71936775..3c4b608a 100644 --- a/test/test_type.py +++ b/test/test_type.py @@ -1,145 +1,328 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2018-2026) import io -import unittest +import sys import pytest -try: - import xxhash -except ImportError: - xxhash = None - import orjson +from .util import SUPPORTS_BYTEARRAY, SUPPORTS_MEMORYVIEW + -class TypeTests(unittest.TestCase): +class TestType: def test_fragment(self): """ orjson.JSONDecodeError on fragments """ for val in ("n", "{", "[", "t"): - self.assertRaises(orjson.JSONDecodeError, orjson.loads, val) + pytest.raises(orjson.JSONDecodeError, orjson.loads, val) def test_invalid(self): """ orjson.JSONDecodeError on invalid """ for val in ('{"age", 44}', "[31337,]", "[,31337]", "[]]", "[,]"): - self.assertRaises(orjson.JSONDecodeError, orjson.loads, val) + pytest.raises(orjson.JSONDecodeError, orjson.loads, val) def test_str(self): """ str """ - for (obj, ref) in (("blah", b'"blah"'), ("東京", b'"\xe6\x9d\xb1\xe4\xba\xac"')): - self.assertEqual(orjson.dumps(obj), ref) - self.assertEqual(orjson.loads(ref), obj) + for obj, ref in (("blah", b'"blah"'), ("東京", b'"\xe6\x9d\xb1\xe4\xba\xac"')): + assert orjson.dumps(obj) == ref + assert orjson.loads(ref) == obj def test_str_latin1(self): """ str latin1 """ - self.assertEqual(orjson.loads(orjson.dumps("üýþÿ")), "üýþÿ") + assert orjson.loads(orjson.dumps("üýþÿ")) == "üýþÿ" def test_str_long(self): """ str long """ for obj in ("aaaa" * 1024, "üýþÿ" * 1024, "好" * 1024, "�" * 1024): - self.assertEqual(orjson.loads(orjson.dumps(obj)), obj) + assert orjson.loads(orjson.dumps(obj)) == obj + + def test_str_2mib(self): + ref = '🐈🐈🐈🐈🐈"üýa0s9999🐈🐈🐈🐈🐈9\0999\\9999' * 1024 * 50 + assert orjson.loads(orjson.dumps(ref)) == ref def test_str_very_long(self): """ str long enough to trigger overflow in bytecount """ for obj in ("aaaa" * 20000, "üýþÿ" * 20000, "好" * 20000, "�" * 20000): - self.assertEqual(orjson.loads(orjson.dumps(obj)), obj) + assert orjson.loads(orjson.dumps(obj)) == obj def test_str_replacement(self): """ str roundtrip � """ - self.assertEqual(orjson.dumps("�"), b'"\xef\xbf\xbd"') - self.assertEqual(orjson.loads(b'"\xef\xbf\xbd"'), "�") + assert orjson.dumps("�") == b'"\xef\xbf\xbd"' + assert orjson.loads(b'"\xef\xbf\xbd"') == "�" + + def test_str_trailing_4_byte(self): + ref = "うぞ〜😏🙌" + assert orjson.loads(orjson.dumps(ref)) == ref + + def test_str_ascii_control(self): + """ + worst case format_escaped_str_with_escapes() allocation + """ + ref = "\x01\x1f" * 1024 * 16 + assert orjson.loads(orjson.dumps(ref)) == ref + assert orjson.loads(orjson.dumps(ref, option=orjson.OPT_INDENT_2)) == ref + + def test_str_escape_quote_0(self): + assert orjson.dumps('"aaaaaaabb') == b'"\\"aaaaaaabb"' + + def test_str_escape_quote_1(self): + assert orjson.dumps('a"aaaaaabb') == b'"a\\"aaaaaabb"' + + def test_str_escape_quote_2(self): + assert orjson.dumps('aa"aaaaabb') == b'"aa\\"aaaaabb"' + + def test_str_escape_quote_3(self): + assert orjson.dumps('aaa"aaaabb') == b'"aaa\\"aaaabb"' + + def test_str_escape_quote_4(self): + assert orjson.dumps('aaaa"aaabb') == b'"aaaa\\"aaabb"' + + def test_str_escape_quote_5(self): + assert orjson.dumps('aaaaa"aabb') == b'"aaaaa\\"aabb"' + + def test_str_escape_quote_6(self): + assert orjson.dumps('aaaaaa"abb') == b'"aaaaaa\\"abb"' + + def test_str_escape_quote_7(self): + assert orjson.dumps('aaaaaaa"bb') == b'"aaaaaaa\\"bb"' + + def test_str_escape_quote_8(self): + assert orjson.dumps('aaaaaaaab"') == b'"aaaaaaaab\\""' + + def test_str_escape_quote_multi(self): + assert ( + orjson.dumps('aa"aaaaabbbbbbbbbbbbbbbbbbbb"bb') + == b'"aa\\"aaaaabbbbbbbbbbbbbbbbbbbb\\"bb"' + ) + + def test_str_escape_quote_buffer(self): + orjson.dumps(['"' * 4096] * 1024) + + def test_str_escape_backslash_0(self): + assert orjson.dumps("\\aaaaaaabb") == b'"\\\\aaaaaaabb"' + + def test_str_escape_backslash_1(self): + assert orjson.dumps("a\\aaaaaabb") == b'"a\\\\aaaaaabb"' + + def test_str_escape_backslash_2(self): + assert orjson.dumps("aa\\aaaaabb") == b'"aa\\\\aaaaabb"' + + def test_str_escape_backslash_3(self): + assert orjson.dumps("aaa\\aaaabb") == b'"aaa\\\\aaaabb"' + + def test_str_escape_backslash_4(self): + assert orjson.dumps("aaaa\\aaabb") == b'"aaaa\\\\aaabb"' + + def test_str_escape_backslash_5(self): + assert orjson.dumps("aaaaa\\aabb") == b'"aaaaa\\\\aabb"' + + def test_str_escape_backslash_6(self): + assert orjson.dumps("aaaaaa\\abb") == b'"aaaaaa\\\\abb"' + + def test_str_escape_backslash_7(self): + assert orjson.dumps("aaaaaaa\\bb") == b'"aaaaaaa\\\\bb"' + + def test_str_escape_backslash_8(self): + assert orjson.dumps("aaaaaaaab\\") == b'"aaaaaaaab\\\\"' + + def test_str_escape_backslash_multi(self): + assert ( + orjson.dumps("aa\\aaaaabbbbbbbbbbbbbbbbbbbb\\bb") + == b'"aa\\\\aaaaabbbbbbbbbbbbbbbbbbbb\\\\bb"' + ) + + def test_str_escape_backslash_buffer(self): + orjson.dumps(["\\" * 4096] * 1024) + + def test_str_escape_x32_0(self): + assert orjson.dumps("\taaaaaaabb") == b'"\\taaaaaaabb"' + + def test_str_escape_x32_1(self): + assert orjson.dumps("a\taaaaaabb") == b'"a\\taaaaaabb"' + + def test_str_escape_x32_2(self): + assert orjson.dumps("aa\taaaaabb") == b'"aa\\taaaaabb"' + + def test_str_escape_x32_3(self): + assert orjson.dumps("aaa\taaaabb") == b'"aaa\\taaaabb"' + + def test_str_escape_x32_4(self): + assert orjson.dumps("aaaa\taaabb") == b'"aaaa\\taaabb"' + + def test_str_escape_x32_5(self): + assert orjson.dumps("aaaaa\taabb") == b'"aaaaa\\taabb"' + + def test_str_escape_x32_6(self): + assert orjson.dumps("aaaaaa\tabb") == b'"aaaaaa\\tabb"' + + def test_str_escape_x32_7(self): + assert orjson.dumps("aaaaaaa\tbb") == b'"aaaaaaa\\tbb"' + + def test_str_escape_x32_8(self): + assert orjson.dumps("aaaaaaaab\t") == b'"aaaaaaaab\\t"' + + def test_str_escape_x32_multi(self): + assert ( + orjson.dumps("aa\taaaaabbbbbbbbbbbbbbbbbbbb\tbb") + == b'"aa\\taaaaabbbbbbbbbbbbbbbbbbbb\\tbb"' + ) + + def test_str_escape_x32_buffer(self): + orjson.dumps(["\t" * 4096] * 1024) + + def test_str_emoji(self): + ref = "®️" + assert orjson.loads(orjson.dumps(ref)) == ref + + def test_str_emoji_escape(self): + ref = '/"®️/"' + assert orjson.loads(orjson.dumps(ref)) == ref + + def test_very_long_list(self): + orjson.dumps([[]] * 1024 * 16) + + def test_very_long_list_pretty(self): + orjson.dumps([[]] * 1024 * 16, option=orjson.OPT_INDENT_2) + + def test_very_long_dict(self): + orjson.dumps([{}] * 1024 * 16) + + def test_very_long_dict_pretty(self): + orjson.dumps([{}] * 1024 * 16, option=orjson.OPT_INDENT_2) + + def test_very_long_str_empty(self): + orjson.dumps([""] * 1024 * 16) + + def test_very_long_str_empty_pretty(self): + orjson.dumps([""] * 1024 * 16, option=orjson.OPT_INDENT_2) + + def test_very_long_str_not_empty(self): + orjson.dumps(["a"] * 1024 * 16) + + def test_very_long_str_not_empty_pretty(self): + orjson.dumps(["a"] * 1024 * 16, option=orjson.OPT_INDENT_2) + + def test_very_long_bool(self): + orjson.dumps([True] * 1024 * 16) + + def test_very_long_bool_pretty(self): + orjson.dumps([True] * 1024 * 16, option=orjson.OPT_INDENT_2) + + def test_very_long_int(self): + orjson.dumps([(2**64) - 1] * 1024 * 16) + + def test_very_long_int_pretty(self): + orjson.dumps([(2**64) - 1] * 1024 * 16, option=orjson.OPT_INDENT_2) + + def test_very_long_float(self): + orjson.dumps([sys.float_info.max] * 1024 * 16) + + def test_very_long_float_pretty(self): + orjson.dumps([sys.float_info.max] * 1024 * 16, option=orjson.OPT_INDENT_2) def test_str_surrogates_loads(self): """ str unicode surrogates loads() """ - self.assertRaises(orjson.JSONDecodeError, orjson.loads, '"\ud800"') - self.assertRaises(orjson.JSONDecodeError, orjson.loads, '"\ud83d\ude80"') - self.assertRaises(orjson.JSONDecodeError, orjson.loads, '"\udcff"') - self.assertRaises( - orjson.JSONDecodeError, orjson.loads, b'"\xed\xa0\xbd\xed\xba\x80"' + pytest.raises(orjson.JSONDecodeError, orjson.loads, '"\ud800"') + pytest.raises(orjson.JSONDecodeError, orjson.loads, '"\ud83d\ude80"') + pytest.raises(orjson.JSONDecodeError, orjson.loads, '"\udcff"') + pytest.raises( + orjson.JSONDecodeError, + orjson.loads, + b'"\xed\xa0\xbd\xed\xba\x80"', ) # \ud83d\ude80 def test_str_surrogates_dumps(self): """ str unicode surrogates dumps() """ - self.assertRaises(orjson.JSONEncodeError, orjson.dumps, "\ud800") - self.assertRaises(orjson.JSONEncodeError, orjson.dumps, "\ud83d\ude80") - self.assertRaises(orjson.JSONEncodeError, orjson.dumps, "\udcff") - self.assertRaises(orjson.JSONEncodeError, orjson.dumps, {"\ud83d\ude80": None}) - self.assertRaises( - orjson.JSONEncodeError, orjson.dumps, b"\xed\xa0\xbd\xed\xba\x80" + pytest.raises(orjson.JSONEncodeError, orjson.dumps, "\ud800") + pytest.raises(orjson.JSONEncodeError, orjson.dumps, "\ud83d\ude80") + pytest.raises(orjson.JSONEncodeError, orjson.dumps, "\udcff") + pytest.raises(orjson.JSONEncodeError, orjson.dumps, {"\ud83d\ude80": None}) + pytest.raises( + orjson.JSONEncodeError, + orjson.dumps, + b"\xed\xa0\xbd\xed\xba\x80", ) # \ud83d\ude80 - @pytest.mark.skipif( - xxhash is None, reason="xxhash install broken on win, python3.9, Azure" - ) - def test_str_ascii(self): - """ - str is ASCII but not compact - """ - digest = xxhash.xxh32_hexdigest("12345") - for _ in range(2): - self.assertEqual(orjson.dumps(digest), b'"b30d56b4"') - def test_bytes_dumps(self): """ bytes dumps not supported """ - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps([b"a"]) def test_bytes_loads(self): """ bytes loads """ - self.assertEqual(orjson.loads(b"[]"), []) + assert orjson.loads(b"[]") == [] + @pytest.mark.skipif(SUPPORTS_BYTEARRAY is False, reason="bytearray") def test_bytearray_loads(self): """ bytearray loads """ arr = bytearray() arr.extend(b"[]") - self.assertEqual(orjson.loads(arr), []) + assert orjson.loads(arr) == [] - def test_memoryview_loads(self): + @pytest.mark.skipif(SUPPORTS_MEMORYVIEW is False, reason="memoryview") + def test_memoryview_loads_supported(self): """ - memoryview loads + memoryview loads supported """ - arr = bytearray() - arr.extend(b"[]") - self.assertEqual(orjson.loads(memoryview(arr)), []) + assert orjson.loads(memoryview(b"[]")) == [] - def test_bytesio_loads(self): + @pytest.mark.skipif(SUPPORTS_MEMORYVIEW is True, reason="memoryview") + def test_memoryview_loads_unsupported(self): """ - memoryview loads + memoryview loads unsupported + """ + with pytest.raises(orjson.JSONDecodeError): + orjson.loads(memoryview(b"[]")) + + @pytest.mark.skipif(SUPPORTS_BYTEARRAY is False, reason="bytearray") + def test_bytesio_loads_supported(self): + """ + BytesIO loads supported + """ + arr = io.BytesIO(b"[]") + assert orjson.loads(arr.getbuffer()) == [] + + @pytest.mark.skipif(SUPPORTS_BYTEARRAY is True, reason="bytearray") + def test_bytesio_loads_unsupported(self): + """ + BytesIO loads unsupported """ arr = io.BytesIO(b"[]") - self.assertEqual(orjson.loads(arr.getbuffer()), []) + with pytest.raises(orjson.JSONDecodeError): + orjson.loads(arr.getbuffer()) def test_bool(self): """ bool """ - for (obj, ref) in ((True, "true"), (False, "false")): - self.assertEqual(orjson.dumps(obj), ref.encode("utf-8")) - self.assertEqual(orjson.loads(ref), obj) + for obj, ref in ((True, "true"), (False, "false")): + assert orjson.dumps(obj) == ref.encode("utf-8") + assert orjson.loads(ref) == obj def test_bool_true_array(self): """ @@ -147,8 +330,8 @@ def test_bool_true_array(self): """ obj = [True] * 256 ref = ("[" + ("true," * 255) + "true]").encode("utf-8") - self.assertEqual(orjson.dumps(obj), ref) - self.assertEqual(orjson.loads(ref), obj) + assert orjson.dumps(obj) == ref + assert orjson.loads(ref) == obj def test_bool_false_array(self): """ @@ -156,8 +339,8 @@ def test_bool_false_array(self): """ obj = [False] * 256 ref = ("[" + ("false," * 255) + "false]").encode("utf-8") - self.assertEqual(orjson.dumps(obj), ref) - self.assertEqual(orjson.loads(ref), obj) + assert orjson.dumps(obj) == ref + assert orjson.loads(ref) == obj def test_none(self): """ @@ -165,8 +348,17 @@ def test_none(self): """ obj = None ref = "null" - self.assertEqual(orjson.dumps(obj), ref.encode("utf-8")) - self.assertEqual(orjson.loads(ref), obj) + assert orjson.dumps(obj) == ref.encode("utf-8") + assert orjson.loads(ref) == obj + + def test_int(self): + """ + int compact and non-compact + """ + obj = [-5000, -1000, -10, -5, -2, -1, 0, 1, 2, 5, 10, 1000, 50000] + ref = b"[-5000,-1000,-10,-5,-2,-1,0,1,2,5,10,1000,50000]" + assert orjson.dumps(obj) == ref + assert orjson.loads(ref) == obj def test_null_array(self): """ @@ -174,41 +366,41 @@ def test_null_array(self): """ obj = [None] * 256 ref = ("[" + ("null," * 255) + "null]").encode("utf-8") - self.assertEqual(orjson.dumps(obj), ref) - self.assertEqual(orjson.loads(ref), obj) + assert orjson.dumps(obj) == ref + assert orjson.loads(ref) == obj def test_nan_dumps(self): """ NaN serializes to null """ - self.assertEqual(orjson.dumps(float("NaN")), b"null") + assert orjson.dumps(float("NaN")) == b"null" def test_nan_loads(self): """ NaN is not valid JSON """ - with self.assertRaises(orjson.JSONDecodeError): + with pytest.raises(orjson.JSONDecodeError): orjson.loads("[NaN]") - with self.assertRaises(orjson.JSONDecodeError): + with pytest.raises(orjson.JSONDecodeError): orjson.loads("[nan]") def test_infinity_dumps(self): """ Infinity serializes to null """ - self.assertEqual(orjson.dumps(float("Infinity")), b"null") + assert orjson.dumps(float("Infinity")) == b"null" def test_infinity_loads(self): """ Infinity, -Infinity is not valid JSON """ - with self.assertRaises(orjson.JSONDecodeError): + with pytest.raises(orjson.JSONDecodeError): orjson.loads("[infinity]") - with self.assertRaises(orjson.JSONDecodeError): + with pytest.raises(orjson.JSONDecodeError): orjson.loads("[Infinity]") - with self.assertRaises(orjson.JSONDecodeError): + with pytest.raises(orjson.JSONDecodeError): orjson.loads("[-Infinity]") - with self.assertRaises(orjson.JSONDecodeError): + with pytest.raises(orjson.JSONDecodeError): orjson.loads("[-infinity]") def test_int_53(self): @@ -216,18 +408,17 @@ def test_int_53(self): int 53-bit """ for val in (9007199254740991, -9007199254740991): - self.assertEqual(orjson.loads(str(val)), val) - self.assertEqual( - orjson.dumps(val, option=orjson.OPT_STRICT_INTEGER), - str(val).encode("utf-8"), - ) + assert orjson.loads(str(val)) == val + assert orjson.dumps(val, option=orjson.OPT_STRICT_INTEGER) == str( + val, + ).encode("utf-8") def test_int_53_exc(self): """ int 53-bit exception on 64-bit """ for val in (9007199254740992, -9007199254740992): - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(val, option=orjson.OPT_STRICT_INTEGER) def test_int_53_exc_usize(self): @@ -235,95 +426,103 @@ def test_int_53_exc_usize(self): int 53-bit exception on 64-bit usize """ for val in (9223372036854775808, 18446744073709551615): - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(val, option=orjson.OPT_STRICT_INTEGER) + def test_int_53_exc_128(self): + """ + int 53-bit exception on 128-bit + """ + val = 2**65 + with pytest.raises(orjson.JSONEncodeError): + orjson.dumps(val, option=orjson.OPT_STRICT_INTEGER) + def test_int_64(self): """ int 64-bit """ for val in (9223372036854775807, -9223372036854775807): - self.assertEqual(orjson.loads(str(val)), val) - self.assertEqual(orjson.dumps(val), str(val).encode("utf-8")) + assert orjson.loads(str(val)) == val + assert orjson.dumps(val) == str(val).encode("utf-8") def test_uint_64(self): """ uint 64-bit """ for val in (0, 9223372036854775808, 18446744073709551615): - self.assertEqual(orjson.loads(str(val)), val) - self.assertEqual(orjson.dumps(val), str(val).encode("utf-8")) + assert orjson.loads(str(val)) == val + assert orjson.dumps(val) == str(val).encode("utf-8") def test_int_128(self): """ int 128-bit """ for val in (18446744073709551616, -9223372036854775809): - self.assertRaises(orjson.JSONEncodeError, orjson.dumps, val) + pytest.raises(orjson.JSONEncodeError, orjson.dumps, val) def test_float(self): """ float """ - self.assertEqual(-1.1234567893, orjson.loads("-1.1234567893")) - self.assertEqual(-1.234567893, orjson.loads("-1.234567893")) - self.assertEqual(-1.34567893, orjson.loads("-1.34567893")) - self.assertEqual(-1.4567893, orjson.loads("-1.4567893")) - self.assertEqual(-1.567893, orjson.loads("-1.567893")) - self.assertEqual(-1.67893, orjson.loads("-1.67893")) - self.assertEqual(-1.7893, orjson.loads("-1.7893")) - self.assertEqual(-1.893, orjson.loads("-1.893")) - self.assertEqual(-1.3, orjson.loads("-1.3")) - - self.assertEqual(1.1234567893, orjson.loads("1.1234567893")) - self.assertEqual(1.234567893, orjson.loads("1.234567893")) - self.assertEqual(1.34567893, orjson.loads("1.34567893")) - self.assertEqual(1.4567893, orjson.loads("1.4567893")) - self.assertEqual(1.567893, orjson.loads("1.567893")) - self.assertEqual(1.67893, orjson.loads("1.67893")) - self.assertEqual(1.7893, orjson.loads("1.7893")) - self.assertEqual(1.893, orjson.loads("1.893")) - self.assertEqual(1.3, orjson.loads("1.3")) + assert -1.1234567893 == orjson.loads("-1.1234567893") + assert -1.234567893 == orjson.loads("-1.234567893") + assert -1.34567893 == orjson.loads("-1.34567893") + assert -1.4567893 == orjson.loads("-1.4567893") + assert -1.567893 == orjson.loads("-1.567893") + assert -1.67893 == orjson.loads("-1.67893") + assert -1.7893 == orjson.loads("-1.7893") + assert -1.893 == orjson.loads("-1.893") + assert -1.3 == orjson.loads("-1.3") + + assert 1.1234567893 == orjson.loads("1.1234567893") + assert 1.234567893 == orjson.loads("1.234567893") + assert 1.34567893 == orjson.loads("1.34567893") + assert 1.4567893 == orjson.loads("1.4567893") + assert 1.567893 == orjson.loads("1.567893") + assert 1.67893 == orjson.loads("1.67893") + assert 1.7893 == orjson.loads("1.7893") + assert 1.893 == orjson.loads("1.893") + assert 1.3 == orjson.loads("1.3") def test_float_precision_loads(self): """ float precision loads() """ - self.assertEqual(orjson.loads("31.245270191439438"), 31.245270191439438) - self.assertEqual(orjson.loads("-31.245270191439438"), -31.245270191439438) - self.assertEqual(orjson.loads("121.48791951161945"), 121.48791951161945) - self.assertEqual(orjson.loads("-121.48791951161945"), -121.48791951161945) - self.assertEqual(orjson.loads("100.78399658203125"), 100.78399658203125) - self.assertEqual(orjson.loads("-100.78399658203125"), -100.78399658203125) + assert orjson.loads("31.245270191439438") == 31.245270191439438 + assert orjson.loads("-31.245270191439438") == -31.245270191439438 + assert orjson.loads("121.48791951161945") == 121.48791951161945 + assert orjson.loads("-121.48791951161945") == -121.48791951161945 + assert orjson.loads("100.78399658203125") == 100.78399658203125 + assert orjson.loads("-100.78399658203125") == -100.78399658203125 def test_float_precision_dumps(self): """ float precision dumps() """ - self.assertEqual(orjson.dumps(31.245270191439438), b"31.245270191439438") - self.assertEqual(orjson.dumps(-31.245270191439438), b"-31.245270191439438") - self.assertEqual(orjson.dumps(121.48791951161945), b"121.48791951161945") - self.assertEqual(orjson.dumps(-121.48791951161945), b"-121.48791951161945") - self.assertEqual(orjson.dumps(100.78399658203125), b"100.78399658203125") - self.assertEqual(orjson.dumps(-100.78399658203125), b"-100.78399658203125") + assert orjson.dumps(31.245270191439438) == b"31.245270191439438" + assert orjson.dumps(-31.245270191439438) == b"-31.245270191439438" + assert orjson.dumps(121.48791951161945) == b"121.48791951161945" + assert orjson.dumps(-121.48791951161945) == b"-121.48791951161945" + assert orjson.dumps(100.78399658203125) == b"100.78399658203125" + assert orjson.dumps(-100.78399658203125) == b"-100.78399658203125" def test_float_edge(self): """ float edge cases """ - self.assertEqual(orjson.dumps(0.8701), b"0.8701") + assert orjson.dumps(0.8701) == b"0.8701" - self.assertEqual(orjson.loads("0.8701"), 0.8701) - self.assertEqual( - orjson.loads("0.0000000000000000000000000000000000000000000000000123e50"), - 1.23, + assert orjson.loads("0.8701") == 0.8701 + assert ( + orjson.loads("0.0000000000000000000000000000000000000000000000000123e50") + == 1.23 ) - self.assertEqual(orjson.loads("0.4e5"), 40000.0) - self.assertEqual(orjson.loads("0.00e-00"), 0.0) - self.assertEqual(orjson.loads("0.4e-001"), 0.04) - self.assertEqual(orjson.loads("0.123456789e-12"), 1.23456789e-13) - self.assertEqual(orjson.loads("1.234567890E+34"), 1.23456789e34) - self.assertEqual(orjson.loads("23456789012E66"), 2.3456789012e76) + assert orjson.loads("0.4e5") == 40000.0 + assert orjson.loads("0.00e-00") == 0.0 + assert orjson.loads("0.4e-001") == 0.04 + assert orjson.loads("0.123456789e-12") == 1.23456789e-13 + assert orjson.loads("1.234567890E+34") == 1.23456789e34 + assert orjson.loads("23456789012E66") == 2.3456789012e76 def test_float_notation(self): """ @@ -331,8 +530,8 @@ def test_float_notation(self): """ for val in ("1.337E40", "1.337e+40", "1337e40", "1.337E-4"): obj = orjson.loads(val) - self.assertEqual(obj, float(val)) - self.assertEqual(orjson.dumps(val), ('"%s"' % val).encode("utf-8")) + assert obj == float(val) + assert orjson.dumps(val) == (f'"{val}"').encode("utf-8") def test_list(self): """ @@ -340,8 +539,8 @@ def test_list(self): """ obj = ["a", "😊", True, {"b": 1.1}, 2] ref = '["a","😊",true,{"b":1.1},2]' - self.assertEqual(orjson.dumps(obj), ref.encode("utf-8")) - self.assertEqual(orjson.loads(ref), obj) + assert orjson.dumps(obj) == ref.encode("utf-8") + assert orjson.loads(ref) == obj def test_tuple(self): """ @@ -349,85 +548,12 @@ def test_tuple(self): """ obj = ("a", "😊", True, {"b": 1.1}, 2) ref = '["a","😊",true,{"b":1.1},2]' - self.assertEqual(orjson.dumps(obj), ref.encode("utf-8")) - self.assertEqual(orjson.loads(ref), list(obj)) - - def test_dict(self): - """ - dict - """ - obj = {"key": "value"} - ref = '{"key":"value"}' - self.assertEqual(orjson.dumps(obj), ref.encode("utf-8")) - self.assertEqual(orjson.loads(ref), obj) - - def test_dict_duplicate_loads(self): - self.assertEqual(orjson.loads(b'{"1":true,"1":false}'), {"1": False}) - - def test_dict_large(self): - """ - dict with >512 keys - """ - obj = {"key_%s" % idx: "value" for idx in range(513)} - self.assertEqual(len(obj), 513) - self.assertEqual(orjson.loads(orjson.dumps(obj)), obj) - - def test_dict_large_keys(self): - """ - dict with keys too large to cache - """ - obj = { - "keeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeey": "value" - } - ref = '{"keeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeey":"value"}' - self.assertEqual(orjson.dumps(obj), ref.encode("utf-8")) - self.assertEqual(orjson.loads(ref), obj) - - def test_dict_unicode(self): - """ - dict unicode keys - """ - obj = {"🐈": "value"} - ref = b'{"\xf0\x9f\x90\x88":"value"}' - self.assertEqual(orjson.dumps(obj), ref) - self.assertEqual(orjson.loads(ref), obj) - self.assertEqual(orjson.loads(ref)["🐈"], "value") - - def test_dict_invalid_key_dumps(self): - """ - dict invalid key dumps() - """ - with self.assertRaises(orjson.JSONEncodeError): - orjson.dumps({1: "value"}) - with self.assertRaises(orjson.JSONEncodeError): - orjson.dumps({b"key": "value"}) - - def test_dict_invalid_key_loads(self): - """ - dict invalid key loads() - """ - with self.assertRaises(orjson.JSONDecodeError): - orjson.loads('{1:"value"}') - with self.assertRaises(orjson.JSONDecodeError): - orjson.loads('{{"a":true}:true}') + assert orjson.dumps(obj) == ref.encode("utf-8") + assert orjson.loads(ref) == list(obj) def test_object(self): """ object() dumps() """ - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(object()) - - def test_dict_similar_keys(self): - """ - loads() similar keys - - This was a regression in 3.4.2 caused by using - the implementation in wy instead of wyhash. - """ - self.assertEqual( - orjson.loads( - '{"cf_status_firefox67": "---", "cf_status_firefox57": "verified"}' - ), - {"cf_status_firefox57": "verified", "cf_status_firefox67": "---"}, - ) diff --git a/test/test_typeddict.py b/test/test_typeddict.py index 59f6f8c4..74682609 100644 --- a/test/test_typeddict.py +++ b/test/test_typeddict.py @@ -1,16 +1,15 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import unittest +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2019-2023) import orjson try: - from typing import TypedDict + from typing import TypedDict # type: ignore except ImportError: from typing_extensions import TypedDict -class TypedDictTests(unittest.TestCase): +class TestTypedDict: def test_typeddict(self): """ dumps() TypedDict @@ -21,4 +20,4 @@ class TypedDict1(TypedDict): b: int obj = TypedDict1(a="a", b=1) - self.assertEqual(orjson.dumps(obj), b'{"a":"a","b":1}') + assert orjson.dumps(obj) == b'{"a":"a","b":1}' diff --git a/test/test_ujson.py b/test/test_ujson.py deleted file mode 100644 index fac4f126..00000000 --- a/test/test_ujson.py +++ /dev/null @@ -1,364 +0,0 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - - -import json -import math -import unittest - -import orjson - - -class UltraJSONTests(unittest.TestCase): - def test_doubleLongIssue(self): - sut = {"a": -4342969734183514} - encoded = orjson.dumps(sut) - decoded = orjson.loads(encoded) - self.assertEqual(sut, decoded) - encoded = orjson.dumps(sut) - decoded = orjson.loads(encoded) - self.assertEqual(sut, decoded) - - def test_doubleLongDecimalIssue(self): - sut = {"a": -12345678901234.56789012} - encoded = orjson.dumps(sut) - decoded = orjson.loads(encoded) - self.assertEqual(sut, decoded) - encoded = orjson.dumps(sut) - decoded = orjson.loads(encoded) - self.assertEqual(sut, decoded) - - def test_encodeDecodeLongDecimal(self): - sut = {"a": -528656961.4399388} - encoded = orjson.dumps(sut) - orjson.loads(encoded) - - def test_decimalDecodeTest(self): - sut = {"a": 4.56} - encoded = orjson.dumps(sut) - decoded = orjson.loads(encoded) - self.assertAlmostEqual(sut["a"], decoded["a"]) - - def test_encodeDictWithUnicodeKeys(self): - input = { - "key1": "value1", - "key1": "value1", - "key1": "value1", - "key1": "value1", - "key1": "value1", - "key1": "value1", - } - orjson.dumps(input) - - input = { - "بن": "value1", - "بن": "value1", - "بن": "value1", - "بن": "value1", - "بن": "value1", - "بن": "value1", - "بن": "value1", - } - orjson.dumps(input) - - def test_encodeDoubleConversion(self): - input = math.pi - output = orjson.dumps(input) - self.assertEqual(round(input, 5), round(orjson.loads(output), 5)) - self.assertEqual(round(input, 5), round(orjson.loads(output), 5)) - - def test_encodeDoubleNegConversion(self): - input = -math.pi - output = orjson.dumps(input) - - self.assertEqual(round(input, 5), round(orjson.loads(output), 5)) - self.assertEqual(round(input, 5), round(orjson.loads(output), 5)) - - def test_encodeArrayOfNestedArrays(self): - input = [[[[]]]] * 20 - output = orjson.dumps(input) - self.assertEqual(input, orjson.loads(output)) - self.assertEqual(input, orjson.loads(output)) - - def test_encodeArrayOfDoubles(self): - input = [31337.31337, 31337.31337, 31337.31337, 31337.31337] * 10 - output = orjson.dumps(input) - self.assertEqual(input, orjson.loads(output)) - self.assertEqual(input, orjson.loads(output)) - - def test_encodeStringConversion2(self): - input = "A string \\ / \b \f \n \r \t" - output = orjson.dumps(input) - self.assertEqual(input, orjson.loads(output)) - self.assertEqual(output, b'"A string \\\\ / \\b \\f \\n \\r \\t"') - self.assertEqual(input, orjson.loads(output)) - - def test_decodeUnicodeConversion(self): - pass - - def test_encodeUnicodeConversion1(self): - input = "Räksmörgås اسامة بن محمد بن عوض بن لادن" - enc = orjson.dumps(input) - dec = orjson.loads(enc) - self.assertEqual(enc, orjson.dumps(input)) - self.assertEqual(dec, orjson.loads(enc)) - - def test_encodeControlEscaping(self): - input = "\x19" - enc = orjson.dumps(input) - dec = orjson.loads(enc) - self.assertEqual(input, dec) - self.assertEqual(enc, orjson.dumps(input)) - - def test_encodeUnicodeConversion2(self): - input = "\xe6\x97\xa5\xd1\x88" - enc = orjson.dumps(input) - dec = orjson.loads(enc) - self.assertEqual(enc, orjson.dumps(input)) - self.assertEqual(dec, orjson.loads(enc)) - - def test_encodeUnicodeSurrogatePair(self): - input = "\xf0\x90\x8d\x86" - enc = orjson.dumps(input) - dec = orjson.loads(enc) - - self.assertEqual(enc, orjson.dumps(input)) - self.assertEqual(dec, orjson.loads(enc)) - - def test_encodeUnicode4BytesUTF8(self): - input = "\xf0\x91\x80\xb0TRAILINGNORMAL" - enc = orjson.dumps(input) - dec = orjson.loads(enc) - - self.assertEqual(enc, orjson.dumps(input)) - self.assertEqual(dec, orjson.loads(enc)) - - def test_encodeUnicode4BytesUTF8Highest(self): - input = "\xf3\xbf\xbf\xbfTRAILINGNORMAL" - enc = orjson.dumps(input) - dec = orjson.loads(enc) - - self.assertEqual(enc, orjson.dumps(input)) - self.assertEqual(dec, orjson.loads(enc)) - - # Characters outside of Basic Multilingual Plane(larger than - # 16 bits) are represented as \UXXXXXXXX in python but should be encoded - # as \uXXXX\uXXXX in orjson. - def testEncodeUnicodeBMP(self): - s = "\U0001f42e\U0001f42e\U0001F42D\U0001F42D" # 🐮🐮🐭🐭 - orjson.dumps(s) - json.dumps(s) - - self.assertEqual(json.loads(json.dumps(s)), s) - self.assertEqual(orjson.loads(orjson.dumps(s)), s) - - def testEncodeSymbols(self): - s = "\u273f\u2661\u273f" # ✿♡✿ - encoded = orjson.dumps(s) - encoded_json = json.dumps(s) - - decoded = orjson.loads(encoded) - self.assertEqual(s, decoded) - - encoded = orjson.dumps(s) - - # json outputs an unicode object - encoded_json = json.dumps(s, ensure_ascii=False) - self.assertEqual(encoded, encoded_json.encode("utf-8")) - decoded = orjson.loads(encoded) - self.assertEqual(s, decoded) - - def test_encodeArrayInArray(self): - input = [[[[]]]] - output = orjson.dumps(input) - - self.assertEqual(input, orjson.loads(output)) - self.assertEqual(output, orjson.dumps(input)) - self.assertEqual(input, orjson.loads(output)) - - def test_encodeIntConversion(self): - input = 31337 - output = orjson.dumps(input) - self.assertEqual(input, orjson.loads(output)) - self.assertEqual(output, orjson.dumps(input)) - self.assertEqual(input, orjson.loads(output)) - - def test_encodeIntNegConversion(self): - input = -31337 - output = orjson.dumps(input) - self.assertEqual(input, orjson.loads(output)) - self.assertEqual(output, orjson.dumps(input)) - self.assertEqual(input, orjson.loads(output)) - - def test_encodeLongNegConversion(self): - input = -9223372036854775808 - output = orjson.dumps(input) - - orjson.loads(output) - orjson.loads(output) - - self.assertEqual(input, orjson.loads(output)) - self.assertEqual(output, orjson.dumps(input)) - self.assertEqual(input, orjson.loads(output)) - - def test_encodeListConversion(self): - input = [1, 2, 3, 4] - output = orjson.dumps(input) - self.assertEqual(input, orjson.loads(output)) - self.assertEqual(input, orjson.loads(output)) - - def test_encodeDictConversion(self): - input = {"k1": 1, "k2": 2, "k3": 3, "k4": 4} - output = orjson.dumps(input) - self.assertEqual(input, orjson.loads(output)) - self.assertEqual(input, orjson.loads(output)) - self.assertEqual(input, orjson.loads(output)) - - def test_encodeNoneConversion(self): - input = None - output = orjson.dumps(input) - self.assertEqual(input, orjson.loads(output)) - self.assertEqual(output, orjson.dumps(input)) - self.assertEqual(input, orjson.loads(output)) - - def test_encodeTrueConversion(self): - input = True - output = orjson.dumps(input) - self.assertEqual(input, orjson.loads(output)) - self.assertEqual(output, orjson.dumps(input)) - self.assertEqual(input, orjson.loads(output)) - - def test_encodeFalseConversion(self): - input = False - output = orjson.dumps(input) - self.assertEqual(input, orjson.loads(output)) - self.assertEqual(output, orjson.dumps(input)) - self.assertEqual(input, orjson.loads(output)) - - def test_encodeToUTF8(self): - input = b"\xe6\x97\xa5\xd1\x88" - input = input.decode("utf-8") - enc = orjson.dumps(input) - dec = orjson.loads(enc) - self.assertEqual(enc, orjson.dumps(input)) - self.assertEqual(dec, orjson.loads(enc)) - - def test_decodeFromUnicode(self): - input = '{"obj": 31337}' - dec1 = orjson.loads(input) - dec2 = orjson.loads(str(input)) - self.assertEqual(dec1, dec2) - - def test_decodeJibberish(self): - input = "fdsa sda v9sa fdsa" - self.assertRaises(orjson.JSONDecodeError, orjson.loads, input) - - def test_decodeBrokenArrayStart(self): - input = "[" - self.assertRaises(orjson.JSONDecodeError, orjson.loads, input) - - def test_decodeBrokenObjectStart(self): - input = "{" - self.assertRaises(orjson.JSONDecodeError, orjson.loads, input) - - def test_decodeBrokenArrayEnd(self): - input = "]" - self.assertRaises(orjson.JSONDecodeError, orjson.loads, input) - - def test_decodeBrokenObjectEnd(self): - input = "}" - self.assertRaises(orjson.JSONDecodeError, orjson.loads, input) - - def test_decodeObjectDepthTooBig(self): - input = "{" * (1024 * 1024) - self.assertRaises(orjson.JSONDecodeError, orjson.loads, input) - - def test_decodeStringUnterminated(self): - input = '"TESTING' - self.assertRaises(orjson.JSONDecodeError, orjson.loads, input) - - def test_decodeStringUntermEscapeSequence(self): - input = '"TESTING\\"' - self.assertRaises(orjson.JSONDecodeError, orjson.loads, input) - - def test_decodeStringBadEscape(self): - input = '"TESTING\\"' - self.assertRaises(orjson.JSONDecodeError, orjson.loads, input) - - def test_decodeTrueBroken(self): - input = "tru" - self.assertRaises(orjson.JSONDecodeError, orjson.loads, input) - - def test_decodeFalseBroken(self): - input = "fa" - self.assertRaises(orjson.JSONDecodeError, orjson.loads, input) - - def test_decodeNullBroken(self): - input = "n" - self.assertRaises(orjson.JSONDecodeError, orjson.loads, input) - - def test_decodeBrokenDictKeyTypeLeakTest(self): - input = '{{1337:""}}' - for _ in range(1000): - self.assertRaises(orjson.JSONDecodeError, orjson.loads, input) - - def test_decodeBrokenDictLeakTest(self): - input = '{{"key":"}' - for _ in range(1000): - self.assertRaises(orjson.JSONDecodeError, orjson.loads, input) - - def test_decodeBrokenListLeakTest(self): - input = "[[[true" - for _ in range(1000): - self.assertRaises(orjson.JSONDecodeError, orjson.loads, input) - - def test_decodeDictWithNoKey(self): - input = "{{{{31337}}}}" - self.assertRaises(orjson.JSONDecodeError, orjson.loads, input) - - def test_decodeDictWithNoColonOrValue(self): - input = '{{{{"key"}}}}' - self.assertRaises(orjson.JSONDecodeError, orjson.loads, input) - - def test_decodeDictWithNoValue(self): - input = '{{{{"key":}}}}' - self.assertRaises(orjson.JSONDecodeError, orjson.loads, input) - - def test_decodeNumericIntPos(self): - input = "31337" - self.assertEqual(31337, orjson.loads(input)) - - def test_decodeNumericIntNeg(self): - input = "-31337" - self.assertEqual(-31337, orjson.loads(input)) - - def test_encodeNullCharacter(self): - input = "31337 \x00 1337" - output = orjson.dumps(input) - self.assertEqual(input, orjson.loads(output)) - self.assertEqual(output, orjson.dumps(input)) - self.assertEqual(input, orjson.loads(output)) - - input = "\x00" - output = orjson.dumps(input) - self.assertEqual(input, orjson.loads(output)) - self.assertEqual(output, orjson.dumps(input)) - self.assertEqual(input, orjson.loads(output)) - - self.assertEqual(b'" \\u0000\\r\\n "', orjson.dumps(" \u0000\r\n ")) - - def test_decodeNullCharacter(self): - input = '"31337 \\u0000 31337"' - self.assertEqual(orjson.loads(input), json.loads(input)) - - def test_decodeEscape(self): - base = "\u00e5".encode() - quote = b'"' - input = quote + base + quote - self.assertEqual(json.loads(input), orjson.loads(input)) - - def test_decodeBigEscape(self): - for _ in range(10): - base = "\u00e5".encode() - quote = b'"' - input = quote + (base * 1024 * 1024 * 2) + quote - self.assertEqual(json.loads(input), orjson.loads(input)) diff --git a/test/test_uuid.py b/test/test_uuid.py index 3f70e6f3..d40b752a 100644 --- a/test/test_uuid.py +++ b/test/test_uuid.py @@ -1,39 +1,41 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Copyright ijl (2020-2025), Rami Chowdhury (2020) -import unittest import uuid +import pytest + import orjson -class UUIDTests(unittest.TestCase): +class TestUUID: def test_uuid_immutable(self): """ UUID objects are immutable """ val = uuid.uuid4() - with self.assertRaises(TypeError): - val.int = 1 - with self.assertRaises(TypeError): - val.int = None + with pytest.raises(TypeError): + val.int = 1 # type: ignore + with pytest.raises(TypeError): + val.int = None # type: ignore def test_uuid_int(self): """ UUID.int is a 128-bit integer """ val = uuid.UUID("7202d115-7ff3-4c81-a7c1-2a1f067b1ece") - self.assertIsInstance(val.int, int) - self.assertTrue(val.int >= 2**64) - self.assertTrue(val.int < 2**128) - self.assertEqual(val.int, 151546616840194781678008611711208857294) + assert isinstance(val.int, int) + assert val.int >= 2**64 + assert val.int < 2**128 + assert val.int == 151546616840194781678008611711208857294 def test_uuid_overflow(self): """ UUID.int can't trigger errors in _PyLong_AsByteArray """ - with self.assertRaises(ValueError): + with pytest.raises(ValueError): uuid.UUID(int=2**128) - with self.assertRaises(ValueError): + with pytest.raises(ValueError): uuid.UUID(int=-1) def test_uuid_subclass(self): @@ -44,25 +46,25 @@ def test_uuid_subclass(self): class AUUID(uuid.UUID): pass - with self.assertRaises(orjson.JSONEncodeError): + with pytest.raises(orjson.JSONEncodeError): orjson.dumps(AUUID("{12345678-1234-5678-1234-567812345678}")) def test_serializes_withopt(self): """ dumps() accepts deprecated OPT_SERIALIZE_UUID """ - self.assertEqual( + assert ( orjson.dumps( uuid.UUID("7202d115-7ff3-4c81-a7c1-2a1f067b1ece"), option=orjson.OPT_SERIALIZE_UUID, - ), - b'"7202d115-7ff3-4c81-a7c1-2a1f067b1ece"', + ) + == b'"7202d115-7ff3-4c81-a7c1-2a1f067b1ece"' ) def test_nil_uuid(self): - self.assertEqual( - orjson.dumps(uuid.UUID("00000000-0000-0000-0000-000000000000")), - b'"00000000-0000-0000-0000-000000000000"', + assert ( + orjson.dumps(uuid.UUID("00000000-0000-0000-0000-000000000000")) + == b'"00000000-0000-0000-0000-000000000000"' ) def test_all_ways_to_create_uuid_behave_equivalently(self): @@ -75,23 +77,19 @@ def test_all_ways_to_create_uuid_behave_equivalently(self): uuid.UUID("urn:uuid:12345678-1234-5678-1234-567812345678"), uuid.UUID(bytes=b"\x12\x34\x56\x78" * 4), uuid.UUID( - bytes_le=b"\x78\x56\x34\x12\x34\x12\x78\x56" - + b"\x12\x34\x56\x78\x12\x34\x56\x78" + bytes_le=b"\x78\x56\x34\x12\x34\x12\x78\x56\x12\x34\x56\x78\x12\x34\x56\x78", ), uuid.UUID(fields=(0x12345678, 0x1234, 0x5678, 0x12, 0x34, 0x567812345678)), uuid.UUID(int=0x12345678123456781234567812345678), ] result = orjson.dumps(uuids) - canonical_uuids = ['"%s"' % str(u) for u in uuids] - serialized = ("[%s]" % ",".join(canonical_uuids)).encode("utf8") - self.assertEqual(result, serialized) + canonical_uuids = [f'"{u!s}"' for u in uuids] + serialized = ("[{}]".format(",".join(canonical_uuids))).encode("utf8") + assert result == serialized def test_serializes_correctly_with_leading_zeroes(self): instance = uuid.UUID(int=0x00345678123456781234567812345678) - self.assertEqual( - orjson.dumps(instance), - ('"%s"' % str(instance)).encode("utf8"), - ) + assert orjson.dumps(instance) == (f'"{instance!s}"').encode("utf-8") def test_all_uuid_creation_functions_create_serializable_uuids(self): uuids = ( @@ -101,7 +99,4 @@ def test_all_uuid_creation_functions_create_serializable_uuids(self): uuid.uuid5(uuid.NAMESPACE_DNS, "python.org"), ) for val in uuids: - self.assertEqual( - orjson.dumps(val), - f'"{val}"'.encode("utf-8"), - ) + assert orjson.dumps(val) == f'"{val}"'.encode("utf-8") diff --git a/test/util.py b/test/util.py index 248f0a3d..8d5bc168 100644 --- a/test/util.py +++ b/test/util.py @@ -1,25 +1,49 @@ -# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# SPDX-License-Identifier: MPL-2.0 +# Copyright ijl (2018-2026) import lzma import os +import sys +import sysconfig from pathlib import Path -from typing import Any, Dict +from typing import Any + +IS_FREETHREADING = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + +SUPPORTS_MEMORYVIEW = not IS_FREETHREADING + +SUPPORTS_BYTEARRAY = not IS_FREETHREADING + +SUPPORTS_GETREFCOUNT = sys.implementation == "cpython" + +numpy = None # type: ignore +try: + import numpy # type: ignore # noqa: F401 +except ImportError: + pass + +pandas = None # type: ignore +try: + import pandas # type: ignore # noqa: F401 +except ImportError: + pass + +import pytest import orjson -dirname = os.path.join(os.path.dirname(__file__), "../data") +data_dir = os.path.join(os.path.dirname(__file__), "../data") -STR_CACHE: Dict[str, str] = {} +STR_CACHE: dict[str, str] = {} -OBJ_CACHE: Dict[str, Any] = {} +OBJ_CACHE: dict[str, Any] = {} def read_fixture_bytes(filename, subdir=None): if subdir is None: - parts = (dirname, filename) + path = Path(data_dir, filename) else: - parts = (dirname, subdir, filename) - path = Path(*parts) + path = Path(data_dir, subdir, filename) if path.suffix == ".xz": contents = lzma.decompress(path.read_bytes()) else: @@ -28,12 +52,18 @@ def read_fixture_bytes(filename, subdir=None): def read_fixture_str(filename, subdir=None): - if not filename in STR_CACHE: + if filename not in STR_CACHE: STR_CACHE[filename] = read_fixture_bytes(filename, subdir).decode("utf-8") return STR_CACHE[filename] def read_fixture_obj(filename): - if not filename in OBJ_CACHE: + if filename not in OBJ_CACHE: OBJ_CACHE[filename] = orjson.loads(read_fixture_str(filename)) return OBJ_CACHE[filename] + + +needs_data = pytest.mark.skipif( + not Path(data_dir).exists(), + reason="Test depends on ./data dir that contains fixtures", +)