diff options
20 files changed, 123414 insertions, 0 deletions
diff --git a/share/security/advisories/FreeBSD-EN-20:16.vmx.asc b/share/security/advisories/FreeBSD-EN-20:16.vmx.asc new file mode 100644 index 0000000000..d6a8611c2c --- /dev/null +++ b/share/security/advisories/FreeBSD-EN-20:16.vmx.asc @@ -0,0 +1,126 @@ +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA512 + +============================================================================= +FreeBSD-EN-20:16.vmx Errata Notice + The FreeBSD Project + +Topic: vmx driver packet loss and degraded performance + +Category: core +Module: vmx +Announced: 2020-08-05 +Affects: FreeBSD 12.1 +Corrected: 2020-01-20 22:15:33 UTC (stable/12, 12.1-STABLE) + 2020-08-05 17:09:54 UTC (releng/12.1, 12.1-RELEASE-p8) + +For general information regarding FreeBSD Errata Notices and Security +Advisories, including descriptions of the fields above, security +branches, and the following sections, please visit +<URL:https://security.FreeBSD.org/>. + +I. Background + +vmx(4) is a driver for the virtualized network interface device used by +VMware. It implements TCP segmentation offload (TSO), a performance +feature which allows the device to perform TCP segmentation immediately +prior to packet transmission, reducing the amount of work required of +the kernel's TCP implementation. + +II. Problem Description + +vmx(4) in FreeBSD 12.1 contains a bug which causes the driver to set up +transmit descriptors incorrectly when performing TSO. + +III. Impact + +With TSO enabled in vmx(4) interfaces, TCP sessions may hang or +experience degraded performance due to packet loss. + +IV. Workaround + +Using ifconfig(8), TSO can be disabled on vmx(4) interfaces by +specifying "-tso". + +V. Solution + +Upgrade your system to a supported FreeBSD stable or release / security +branch (releng) dated after the correction date and reboot. + +Perform one of the following: + +1) To update your system via a binary patch: + +Systems running a RELEASE version of FreeBSD on the i386 or amd64 +platforms can be updated via the freebsd-update(8) utility: + +# freebsd-update fetch +# freebsd-update install +# shutdown -r +10min "Rebooting for errata update" + +2) To update your system via a source code patch: + +The following patches have been verified to apply to the applicable +FreeBSD release branches. + +a) Download the relevant patch from the location below, and verify the +detached PGP signature using your PGP utility. + +# fetch https://security.FreeBSD.org/patches/EN-20:16/vmx.patch +# fetch https://security.FreeBSD.org/patches/EN-20:16/vmx.patch.asc +# gpg --verify vmx.patch.asc + +b) Apply the patch. Execute the following commands as root: + +# cd /usr/src +# patch < /path/to/patch + +c) Recompile your kernel as described in +<URL:https://www.FreeBSD.org/handbook/kernelconfig.html> and reboot the +system. + +VI. Correction details + +The following list contains the correction revision numbers for each +affected branch. + +Branch/path Revision +- ------------------------------------------------------------------------- +stable/12/ r356932 +releng/12.1/ r363920 +- ------------------------------------------------------------------------- + +To see which files were modified by a particular revision, run the +following command, replacing NNNNNN with the revision number, on a +machine with Subversion installed: + +# svn diff -cNNNNNN --summarize svn://svn.freebsd.org/base + +Or visit the following URL, replacing NNNNNN with the revision number: + +<URL:https://svnweb.freebsd.org/base?view=revision&revision=NNNNNN> + +VII. References + +<URL:https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=236999> + +The latest revision of this advisory is available at +<URL:https://security.FreeBSD.org/advisories/FreeBSD-EN-20:16.vmx.asc> +-----BEGIN PGP SIGNATURE----- + +iQKTBAEBCgB9FiEE/A6HiuWv54gCjWNV05eS9J6n5cIFAl8q63ZfFIAAAAAALgAo +aXNzdWVyLWZwckBub3RhdGlvbnMub3BlbnBncC5maWZ0aGhvcnNlbWFuLm5ldEZD +MEU4NzhBRTVBRkU3ODgwMjhENjM1NUQzOTc5MkY0OUVBN0U1QzIACgkQ05eS9J6n +5cIe2xAAoGWYRnKtTDRNBwKZSXbXSSrNcNv7rsiPGIqHYjn78ZBRypWFlztLzW8G +eAfo7ffcXDN6PfVrhg2ADKBHfOmghOWHvSEoigV8aT9fgBBvBoFoElmvUdLQjn6o +y5ABUoMbwapXSNDQtGEFi5wtBfomcfpZzxVRNTNzzbPCO3gkG3WZ4/0wiS9TXV34 +SMU2xLIeo0qvBGUfHpqTz+6BfCP/rtMCZ2kx6dIVYguGqBkRxkvJA4q4omEeokWz +XDyE32MdosB8DlmozhL+VDCFAB4k328nbO4kY3czdqOmOQ9krDdB176fHfT/+3Zm +6ogK2JvNNY9vZSeB3sqwSkv4j6B8aSb21bEDTopxF93TwsPO9hkIFC1f1ASH0YYP +TtPli/lsTGy1UdhuURNjgK6c5IuWkgeZpuJdX3UDyxDv+TDk8FvlAyR0R9EPsL3t +MoGKy12dsSF+Nkn6K9hmY9nRRpF6dlgHDpWsGQvJ8j8aw2QciVTU60vE47oM47js +v8KIOliq+OzaPWnL420wR0rjXJo3HIQmdyF1sVpLcFRW26QdJ+0No22qB4BLNr9D +zExolxEAlL/6jsrSwBoZdiHGxzxjFUPJBJojARIP2tZSLRlhGFmNJVnzAoPE5KY5 +HuxyDRcLqY0Rmeycs3pdupYd6ze2ViNbJsry7XY9+zbW15e1qNw= +=/2NI +-----END PGP SIGNATURE----- diff --git a/share/security/advisories/FreeBSD-SA-20:21.usb_net.asc b/share/security/advisories/FreeBSD-SA-20:21.usb_net.asc new file mode 100644 index 0000000000..c1d6119b8e --- /dev/null +++ b/share/security/advisories/FreeBSD-SA-20:21.usb_net.asc @@ -0,0 +1,147 @@ +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA512 + +============================================================================= +FreeBSD-SA-20:21.usb_net Security Advisory + The FreeBSD Project + +Topic: Potential memory corruption in USB network device drivers + +Category: core +Module: kernel +Announced: 2020-08-05 +Credits: Ilja van Sprundel, IOActive +Affects: All supported versions of FreeBSD. +Corrected: 2020-06-14 05:25:06 UTC (stable/12, 12.1-STABLE) + 2020-08-05 17:11:18 UTC (releng/12.1, 12.1-RELEASE-p8) + 2020-06-14 05:27:37 UTC (stable/11, 11.4-STABLE) + 2020-08-05 17:11:18 UTC (releng/11.4, 11.4-RELEASE-p2) + 2020-08-05 17:11:18 UTC (releng/11.3, 11.3-RELEASE-p12) +CVE Name: CVE-2020-7459 + +For general information regarding FreeBSD Security Advisories, +including descriptions of the fields above, security branches, and the +following sections, please visit <URL:https://security.FreeBSD.org/>. + +I. Background + +FreeBSD includes a number of USB Ethernet network interface device drivers, +including: + + - smsc(4), supporting SMSC (now Microchip) devices + - muge(4), supporting Microchip devices + - cdceem(4), supporting USB Communication Device Class compatible devices + +II. Problem Description + +A missing length validation code common to these three drivers means that a +malicious USB device could write beyond the end of an allocated network +packet buffer. + +III. Impact + +An attacker with physical access to a USB port and the ability to bring a +network interface up may be able to use a specially crafted USB device to +gain kernel or user-space code execution. + +IV. Workaround + +No workaround is available. Systems with no active (i.e., UP) interface +supported by any of the smsc(4), muge(4), and cdceem(4) drivers are not +vulnerable. + +Exploitation likely requires malicious USB hardware that emulates hardware +supported by one of these device drivers. + +V. Solution + +Upgrade your vulnerable system to a supported FreeBSD stable or +release / security branch (releng) dated after the correction date, +and reboot. + +Perform one of the following: + +1) To update your vulnerable system via a binary patch: + +Systems running a RELEASE version of FreeBSD on the i386 or amd64 +platforms can be updated via the freebsd-update(8) utility: + +# freebsd-update fetch +# freebsd-update install +# shutdown -r +10min "Rebooting for a security update" + +2) To update your vulnerable system via a source code patch: + +The following patches have been verified to apply to the applicable +FreeBSD release branches. + +a) Download the relevant patch from the location below, and verify the +detached PGP signature using your PGP utility. + +[FreeBSD 12.x] +# fetch https://security.FreeBSD.org/patches/SA-20:21/usb_net.12.patch +# fetch https://security.FreeBSD.org/patches/SA-20:21/usb_net.12.patch.asc +# gpg --verify usb_net.12.patch.asc + +[FreeBSD 11.x] +# fetch https://security.FreeBSD.org/patches/SA-20:21/usb_net.11.patch +# fetch https://security.FreeBSD.org/patches/SA-20:21/usb_net.11.patch.asc +# gpg --verify usb_net.11.patch.asc + +b) Apply the patch. Execute the following commands as root: + +# cd /usr/src +# patch < /path/to/patch + +c) Recompile your kernel as described in +<URL:https://www.FreeBSD.org/handbook/kernelconfig.html> and reboot the +system. + +VI. Correction details + +The following list contains the correction revision numbers for each +affected branch. + +Branch/path Revision +- ------------------------------------------------------------------------- +stable/12/ r362166 +releng/12.1/ r363921 +stable/11/ r362167 +releng/11.4/ r363921 +releng/11.3/ r363921 +- ------------------------------------------------------------------------- + +To see which files were modified by a particular revision, run the +following command, replacing NNNNNN with the revision number, on a +machine with Subversion installed: + +# svn diff -cNNNNNN --summarize svn://svn.freebsd.org/base + +Or visit the following URL, replacing NNNNNN with the revision number: + +<URL:https://svnweb.freebsd.org/base?view=revision&revision=NNNNNN> + +VII. References + +<URL:https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-7459> + +The latest revision of this advisory is available at +<URL:https://security.FreeBSD.org/advisories/FreeBSD-SA-20:21.usb_net.asc> +-----BEGIN PGP SIGNATURE----- + +iQKTBAEBCgB9FiEE/A6HiuWv54gCjWNV05eS9J6n5cIFAl8q63dfFIAAAAAALgAo +aXNzdWVyLWZwckBub3RhdGlvbnMub3BlbnBncC5maWZ0aGhvcnNlbWFuLm5ldEZD +MEU4NzhBRTVBRkU3ODgwMjhENjM1NUQzOTc5MkY0OUVBN0U1QzIACgkQ05eS9J6n +5cIp7g//ZGKbxNKWsYEFXwNbBVbtkWMCQbj0V0TG2NReOsbYYhOolfkErZBpeTHt +iFJr5m3WY21a6ks/8up02HJyZ5oZwFFeMOMF/CKibZuym/8L8XgoU0uh/eHLiTvZ +qRf3p43xegcKgatFPggKA0yewNxMNETHI7BeO6+pkwYFQgb7f9GhM4JzC+DmaX6i +kyGcrnYoZzgKo2BGt65VRbvzucw/Su7wL4JutKhZlXDz7kxFv1gxB8Dqk9BBW9QM +EHejzhMCo6T0doYKLuZnz+SI2a/LOcTuNgR+5RnnxsVziLx+8csMNYy3YVw6JNXe +XC+8aJ7Un8BLKDoNJjoZ/J9IygJoaWgUa9+SH0pAtOeWhrfRUgd74ZZWfhiZkK5U +AXgY46c6Ce28TbEHTWgOAQgXRNB7iJgxVo6mTSnDt8t3YWh4t3g/rjHPKHagTNYC +aCd6gcJewb1Pw/8X/7H1FXRtUleHgMaxQ7ec8V5BwcXSexo4xZfq8qQTUbCuRmDg +4GaF8SondVb1TJxHwfq2wWvFhiwMWnRxwwjY6jkxiIjecc5vtrb2bwRq7nmKWciT +uV0jRj9ttP73ftE/zO94avXCbpCfHXMSpwaJMcs8PH+sHYXNhy0awuIped1ANXlh +E2jrNBW85gyKpnjfcAgECFid3Cu1V1xWo1BCTOWJXQjKi2Gaoa0= +=xbKP +-----END PGP SIGNATURE----- diff --git a/share/security/advisories/FreeBSD-SA-20:22.sqlite.asc b/share/security/advisories/FreeBSD-SA-20:22.sqlite.asc new file mode 100644 index 0000000000..838874939c --- /dev/null +++ b/share/security/advisories/FreeBSD-SA-20:22.sqlite.asc @@ -0,0 +1,159 @@ +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA512 + +============================================================================= +FreeBSD-SA-20:22.sqlite Security Advisory + The FreeBSD Project + +Topic: Multiple vulnerabilities in sqlite3 + +Category: contrib +Module: sqlite3 +Announced: 2020-08-05 +Affects: All supported versions of FreeBSD. +Corrected: 2020-06-15 03:10:53 UTC (stable/12, 12.1-STABLE) + 2020-08-05 17:13:08 UTC (releng/12.1, 12.1-RELEASE-p8) + 2020-06-15 03:10:53 UTC (stable/11, 11.4-STABLE) + 2020-08-05 17:13:08 UTC (releng/11.4, 11.4-RELEASE-p2) + 2020-08-05 17:13:08 UTC (releng/11.3, 11.3-RELEASE-p12) +CVE Name: CVE-2020-11655, CVE-2020-11656, CVE-2020-13434, + CVE-2020-13435, CVE-2020-13630, CVE-2020-13631, + CVE-2020-13632 + +For general information regarding FreeBSD Security Advisories, +including descriptions of the fields above, security branches, and the +following sections, please visit <URL:https://security.FreeBSD.org/>. + +I. Background + +SQLite is an SQL database engine in a C library. Programs that link the +SQLite library can have SQL database access without running a separate RDBMS +process. The distribution comes with a standalone command-line access +program (sqlite3) that can be used to administer an SQLite database and which +serves as an example of how to use the SQLite library. + +FreeBSD includes SQLite as a private library for base system usage that is +not generally exposed for third party packages to use. + +II. Problem Description + +Multiple vulnerabilities have been published including improper input +validation (CVE-2020-11655), use after free (CVE-2020-11656, CVE-2020-13630), +integer overflow (CVE-2020-13434), null pointer dereference (CVE-2020-13435, +CVE-2020-13632), and namespace collision (CVE-2020-13631). + +III. Impact + +Malicious SQL statements could crash, hijack processes, or cause data +corruption. + +IV. Workaround + +No workaround is available. The FreeBSD security team is not aware of any +base system components that use SQLite in such a way as to expose these +vulnerabilities to untrusted or remote users, but is updating SQLite out of +an abundance of caution. + +V. Solution + +Upgrade your vulnerable system to a supported FreeBSD stable or +release / security branch (releng) dated after the correction date. + +Perform one of the following: + +1) To update your vulnerable system via a binary patch: + +Systems running a RELEASE version of FreeBSD on the i386 or amd64 +platforms can be updated via the freebsd-update(8) utility: + +# freebsd-update fetch +# freebsd-update install +# shutdown -r +10min "Rebooting for a security update" + +2) To update your vulnerable system via a source code patch: + +The following patches have been verified to apply to the applicable +FreeBSD release branches. + +a) Download the relevant patch from the location below, and verify the +detached PGP signature using your PGP utility. + +[FreeBSD 12.1] +# fetch https://security.FreeBSD.org/patches/SA-20:21/sqlite.12.1.patch +# fetch https://security.FreeBSD.org/patches/SA-20:21/sqlite.12.1.patch.asc +# gpg --verify sqlite.12.1.patch.asc + +[FreeBSD 11.4] +# fetch https://security.FreeBSD.org/patches/SA-20:21/sqlite.11.4.patch +# fetch https://security.FreeBSD.org/patches/SA-20:21/sqlite.11.4.patch.asc +# gpg --verify sqlite.11.4.patch.asc + +[FreeBSD 11.3] +# fetch https://security.FreeBSD.org/patches/SA-20:21/sqlite.11.3.patch +# fetch https://security.FreeBSD.org/patches/SA-20:21/sqlite.11.3.patch.asc +# gpg --verify sqlite.11.3.patch.asc + +b) Apply the patch. Execute the following commands as root: + +# cd /usr/src +# patch < /path/to/patch + +c) Recompile the operating system using buildworld and installworld as +described in <URL:https://www.FreeBSD.org/handbook/makeworld.html>. + +Restart all daemons that use the library, or reboot the system. + +VI. Correction details + +The following list contains the correction revision numbers for each +affected branch. + +Branch/path Revision +- ------------------------------------------------------------------------- +stable/12/ r362190 +releng/12.1/ r363922 +stable/11/ r362190 +releng/11.4/ r363922 +releng/11.3/ r363922 +- ------------------------------------------------------------------------- + +To see which files were modified by a particular revision, run the +following command, replacing NNNNNN with the revision number, on a +machine with Subversion installed: + +# svn diff -cNNNNNN --summarize svn://svn.freebsd.org/base + +Or visit the following URL, replacing NNNNNN with the revision number: + +<URL:https://svnweb.freebsd.org/base?view=revision&revision=NNNNNN> + +VII. References + +<URL:https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-11655> +<URL:https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-11656> +<URL:https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-13434> +<URL:https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-13435> +<URL:https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-13630> +<URL:https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-13631> +<URL:https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-13632> + +The latest revision of this advisory is available at +<URL:https://security.FreeBSD.org/advisories/FreeBSD-SA-20:21.sqlite.asc> +-----BEGIN PGP SIGNATURE----- + +iQKTBAEBCgB9FiEE/A6HiuWv54gCjWNV05eS9J6n5cIFAl8q63dfFIAAAAAALgAo +aXNzdWVyLWZwckBub3RhdGlvbnMub3BlbnBncC5maWZ0aGhvcnNlbWFuLm5ldEZD +MEU4NzhBRTVBRkU3ODgwMjhENjM1NUQzOTc5MkY0OUVBN0U1QzIACgkQ05eS9J6n +5cIJdw/9FSXst26ZsA63KixpJbWMnlzJkSPEPiGQ0SlrfeE7co6qukiAnxKzL9rW +P5ztBbqqcqu/mISIDDRaOp03ex1oAwhYf4cMRCX4v1wZmGfdH6L9s0LwgfFXtT2G +RwucImiAKHwLo7YTxUs54qrSu6gmDUp9u440ac1tVHSEsKzvD557sg4sp248tSZx ++/W667F2xsed4plRvNostfFD1aIbBsgMl0vz4FPZ3dToxrjpeSW+9aHrv6iIgsSA +jheXF/Fol5AABrHrOHc8HbKBsDEsmz/AdwHiX1ngH1dXRRze95YEJy64Ee0C7/Fj +MXlhg3JqctCFXy2e2nTHna5xKd3YW4Gy2b2xquIAg/W9rZRxy1ZwQEOO5R+DyteF +s/YN6oD4jJPsR2uTUMq4Z6q7IKDwb7PT2ncTblxIG7vBs6V6NuM0Yd0cqMdPnEdt +rn7hIaPvvsp3nFYMPhIX9gMRl5K9Vl11BWtfEFv3Egh5c5jA0/LWvMP1DLkpKV8c +lXP5C/cltod7zTAkCk8XxOOCi5fLnP8qPQhAy3etq2dtREMgkHfnIxPSmjqsPgBI +uU4CXW3dDlh9RrsePwls++BItUcueKDUJYBAS98Z+XLxaapjj4R3fYa2ygbyjqno +nprfR9X6QW6MPv0xL9wCTGqwQXrvWxDxlnTPEQ5Ah6eOCKEfWD8= +=nB/p +-----END PGP SIGNATURE----- diff --git a/share/security/advisories/FreeBSD-SA-20:23.sendmsg.asc b/share/security/advisories/FreeBSD-SA-20:23.sendmsg.asc new file mode 100644 index 0000000000..1dc335343c --- /dev/null +++ b/share/security/advisories/FreeBSD-SA-20:23.sendmsg.asc @@ -0,0 +1,146 @@ +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA512 + +============================================================================= +FreeBSD-SA-20:23.sendmsg Security Advisory + The FreeBSD Project + +Topic: sendmsg(2) privilege escalation + +Category: core +Module: kernel compat32 +Announced: 2020-08-05 +Credits: m00nbsd working with Trend Micro Zero Day Initiative +Affects: All supported versions of FreeBSD. +Corrected: 2020-08-05 17:07:13 UTC (stable/12, 12.1-STABLE) + 2020-08-05 17:14:01 UTC (releng/12.1, 12.1-RELEASE-p8) + 2020-08-05 17:08:02 UTC (stable/11, 11.4-STABLE) + 2020-08-05 17:14:01 UTC (releng/11.4, 11.4-RELEASE-p2) + 2020-08-05 17:14:01 UTC (releng/11.3, 11.3-RELEASE-p12) +CVE Name: CVE-2020-7460 + +For general information regarding FreeBSD Security Advisories, +including descriptions of the fields above, security branches, and the +following sections, please visit <URL:https://security.FreeBSD.org/>. + +I. Background + +FreeBSD provides the compat32 subsystem, used to enable execution of 32-bit +binaries on amd64 and other 64-bit platforms. System calls whose parameters +require translation are handled by compat32 before being dispatched to the +native system call handler. + +sendmsg(2) and recvmsg(2) may be used to transmit or receive control messages +whose contents are evaluated by the kernel. Such messages have different +alignment constraints on 32-bit and 64-bit platforms and thus must be translated +by the compat32 subsystem when sendmsg(2) or recvmsg(2) are invoked by a 32-bit +process. + +II. Problem Description + +When handling a 32-bit sendmsg(2) call, the compat32 subsystem copies the +control message to be transmitted (if any) into kernel memory, and adjusts +alignment of control message headers. The code which performs this work +contained a time-of-check to time-of-use (TOCTOU) vulnerability which allows a +malicious userspace program to modify control message headers after they were +validated by the kernel. + +III. Impact + +The TOCTOU bug can be exploited by an unprivileged malicious userspace program +to trigger privilege escalation. + +IV. Workaround + +i386 and other 32-bit platforms are not vulnerable. + +No workaround is available for amd64 or arm64. Kernels compiled without the +COMPAT_FREEBSD32 option are not vulnerable, but this option is configured in +GENERIC kernels. + +V. Solution + +Upgrade your vulnerable system to a supported FreeBSD stable or +release / security branch (releng) dated after the correction date, +and reboot. + +Perform one of the following: + +1) To update your vulnerable system via a binary patch: + +Systems running a RELEASE version of FreeBSD on the i386 or amd64 +platforms can be updated via the freebsd-update(8) utility: + +# freebsd-update fetch +# freebsd-update install +# shutdown -r +10min "Rebooting for a security update" + +2) To update your vulnerable system via a source code patch: + +The following patches have been verified to apply to the applicable +FreeBSD release branches. + +a) Download the relevant patch from the location below, and verify the +detached PGP signature using your PGP utility. + +# fetch https://security.FreeBSD.org/patches/SA-20:23/sendmsg.patch +# fetch https://security.FreeBSD.org/patches/SA-20:23/sendmsg.patch.asc +# gpg --verify sendmsg.patch.asc + +b) Apply the patch. Execute the following commands as root: + +# cd /usr/src +# patch < /path/to/patch + +c) Recompile your kernel as described in +<URL:https://www.FreeBSD.org/handbook/kernelconfig.html> and reboot the +system. + +VI. Correction details + +The following list contains the correction revision numbers for each +affected branch. + +Branch/path Revision +- ------------------------------------------------------------------------- +stable/12/ r363918 +releng/12.1/ r363923 +stable/11/ r363919 +releng/11.4/ r363923 +releng/11.3/ r363923 +- ------------------------------------------------------------------------- + +To see which files were modified by a particular revision, run the +following command, replacing NNNNNN with the revision number, on a +machine with Subversion installed: + +# svn diff -cNNNNNN --summarize svn://svn.freebsd.org/base + +Or visit the following URL, replacing NNNNNN with the revision number: + +<URL:https://svnweb.freebsd.org/base?view=revision&revision=NNNNNN> + +VII. References + +<URL:https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-7460> + +The latest revision of this advisory is available at +<URL:https://security.FreeBSD.org/advisories/FreeBSD-SA-20:23.sendmsg.asc> +-----BEGIN PGP SIGNATURE----- + +iQKTBAEBCgB9FiEE/A6HiuWv54gCjWNV05eS9J6n5cIFAl8q63hfFIAAAAAALgAo +aXNzdWVyLWZwckBub3RhdGlvbnMub3BlbnBncC5maWZ0aGhvcnNlbWFuLm5ldEZD +MEU4NzhBRTVBRkU3ODgwMjhENjM1NUQzOTc5MkY0OUVBN0U1QzIACgkQ05eS9J6n +5cJDxw/+PML4MB46paBDyLvebprXe5Z/FrUN1ybH8YjrJTYFyX+aw8K3hAa2K+PB +kUc3VwbIv0BAylSCgULXdTxx8JKpMnmHcN3Bk8LKdMCvp0gJvkzcAP/a7Kj1EKfY +m04p3/7ka9f7u99n1QX1jnZP2XWEFKOoyWbEJWuDk0+NBW3ICQWqQLoiXaWAS4HD +BrXAqowtyoR1vaMrAjmSyWaFSDFjQeiHw8nxCzRF1E6cKF/rwCt37cnpEGqSCAYi +/ZyB1qy1s67F9hHnZp9+JhffWqUZAuLse8HTWgBG+svpzDrx2gNxE/C/Tui0lYXg +S3akC3DbiySZpP007J8yR5PvytYAbSuECJVVRoC0dukmBabFqSFlacInpfIn363m +fOg1nmq/oRh9MAJzaBKG+N6SD+mP3kvcV9Ad5fOKr4yLQtlwYEYyiN1WbCs0O/ve +fnRIGB9xtibIr1i9IEY7+KNMAH3Di2F0E4ixFPMrBcJiStuZmCTqJRx99QLYtb0G +p9p1bzjPUaWAMDi9mteFu1I+NO836MeLydbCZnSa5KLe+vc1PjP4kSvt6XQ9HFtO +nXMddWxdcus8BmxZ04K5a4WaaSYOiN4e4O72WWuA714io+EWJAEaqleMr7KbYTCv +f1fCmoKxyoFJcHL9z3oOOi5DqrBoFPnE0p/gPGFc8qyTNEbI428= +=tTSr +-----END PGP SIGNATURE----- diff --git a/share/security/patches/EN-20:16/vmx.patch b/share/security/patches/EN-20:16/vmx.patch new file mode 100644 index 0000000000..457d9acb47 --- /dev/null +++ b/share/security/patches/EN-20:16/vmx.patch @@ -0,0 +1,11 @@ +--- sys/dev/vmware/vmxnet3/if_vmx.c.orig ++++ sys/dev/vmware/vmxnet3/if_vmx.c +@@ -1320,7 +1320,7 @@ + hdrlen = pi->ipi_ehdrlen + pi->ipi_ip_hlen; + if (pi->ipi_csum_flags & CSUM_TSO) { + sop->offload_mode = VMXNET3_OM_TSO; +- sop->hlen = hdrlen; ++ sop->hlen = hdrlen + pi->ipi_tcp_hlen; + sop->offload_pos = pi->ipi_tso_segsz; + } else if (pi->ipi_csum_flags & (VMXNET3_CSUM_OFFLOAD | + VMXNET3_CSUM_OFFLOAD_IPV6)) { diff --git a/share/security/patches/EN-20:16/vmx.patch.asc b/share/security/patches/EN-20:16/vmx.patch.asc new file mode 100644 index 0000000000..1dd149b6f7 --- /dev/null +++ b/share/security/patches/EN-20:16/vmx.patch.asc @@ -0,0 +1,18 @@ +-----BEGIN PGP SIGNATURE----- + +iQKTBAABCgB9FiEE/A6HiuWv54gCjWNV05eS9J6n5cIFAl8q63dfFIAAAAAALgAo +aXNzdWVyLWZwckBub3RhdGlvbnMub3BlbnBncC5maWZ0aGhvcnNlbWFuLm5ldEZD +MEU4NzhBRTVBRkU3ODgwMjhENjM1NUQzOTc5MkY0OUVBN0U1QzIACgkQ05eS9J6n +5cKCUg//TowAGFMQ0j1V6uIOnxlUWYu1RsFHqrUjh8QSeZmX8oz0B2IWLVzI8mk3 +9R2n4xUmuEVoF9WAR7gxhWuateUjAi4ksg2RdaFVb0Q2p8bQ6Tk2zj9wF5uy+CEQ +2pL0IzrsBc567CQjdhV8JfsHm0wa6BcZ8Pnr7AVVAUPY78XWdvamMupBxwil9zsC +zKCoxZdZk5xDe3kprWu90K5cdCxvvkJgzkVXZFzunqBbrpsjBlL4O5GQyMsFf4Ri +LJHieGwW6UhIAPYEu6tqTf0LanKbBxr/pgB+lD9b4W6YNSi5rEaPcNuBdCp5A/8L +OJjxXU7AUVRFe1YPP0m8FDonIi/5aEOF9VO6Fx5a3P3FBjpM/7CB34gfEv+ZIsrR +o27P7HhAmOdWw8nii50c9ukvhE66gNf3MglDAP6mB8YnSxHMpu5yqVQiGKjoO5hI +scxrlIDHzYWwc22mEJpZSVuuxMo7pFinKT0WwdNq3tSYpErNT4xvE1OT1ZTkRKB7 +bxWqbay7WGP9YHjq1vBgv9tt/iRB27Q0SsVsmGQchhoZANl03jU4wpO8ygAtRd0K +E13BhfaNQLzdByPQPhDLCflMqFHHC9IxKlHg9lVKb/CtDv52yh8/t21IrBy/iqJY +K5Ivn8GY3xOgCwQa5lXE0RzvMLkt4RjFqV++34jowXPyF9KT0ik= +=4RPg +-----END PGP SIGNATURE----- diff --git a/share/security/patches/SA-20:21/usb_net.11.patch b/share/security/patches/SA-20:21/usb_net.11.patch new file mode 100644 index 0000000000..5eb3d46643 --- /dev/null +++ b/share/security/patches/SA-20:21/usb_net.11.patch @@ -0,0 +1,36 @@ +--- sys/dev/usb/net/if_smsc.c.orig ++++ sys/dev/usb/net/if_smsc.c +@@ -970,7 +970,7 @@ + struct mbuf *m; + struct usb_page_cache *pc; + uint32_t rxhdr; +- uint16_t pktlen; ++ int pktlen; + int off; + int actlen; + +@@ -996,6 +996,9 @@ + /* The frame header is always aligned on a 4 byte boundary */ + off = ((off + 0x3) & ~0x3); + ++ if ((off + sizeof(rxhdr)) > actlen) ++ goto tr_setup; ++ + usbd_copy_out(pc, off, &rxhdr, sizeof(rxhdr)); + off += (sizeof(rxhdr) + ETHER_ALIGN); + rxhdr = le32toh(rxhdr); +@@ -1024,7 +1027,13 @@ + if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); + goto tr_setup; + } +- ++ if (pktlen > m->m_len) { ++ smsc_dbg_printf(sc, "buffer too small %d vs %d bytes", ++ pktlen, m->m_len); ++ if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); ++ m_freem(m); ++ goto tr_setup; ++ } + usbd_copy_out(pc, off, mtod(m, uint8_t *), pktlen); + + /* Check if RX TCP/UDP checksumming is being offloaded */ diff --git a/share/security/patches/SA-20:21/usb_net.11.patch.asc b/share/security/patches/SA-20:21/usb_net.11.patch.asc new file mode 100644 index 0000000000..89495945b9 --- /dev/null +++ b/share/security/patches/SA-20:21/usb_net.11.patch.asc @@ -0,0 +1,18 @@ +-----BEGIN PGP SIGNATURE----- + +iQKTBAABCgB9FiEE/A6HiuWv54gCjWNV05eS9J6n5cIFAl8q63dfFIAAAAAALgAo +aXNzdWVyLWZwckBub3RhdGlvbnMub3BlbnBncC5maWZ0aGhvcnNlbWFuLm5ldEZD +MEU4NzhBRTVBRkU3ODgwMjhENjM1NUQzOTc5MkY0OUVBN0U1QzIACgkQ05eS9J6n +5cKxGw//bjW21aF/b5zZn8HaB7ZTah8MLuppHhcKzEfy2T3e95MXwd8t/ZYP/1Th +UwTGxO/mWHIZq9Ky6Y7oS1FUifOsbnaMZjQBioF/+dyRBApczitrIfyyVzL7hBFR +u2stVlIPAoNsa+XKKxxo/jRcHR3RAH3OL7nfN4Bt+sd39lkpWyP6rZxfmD/4Xnmv +YirpypsMFLI1APeIgk0OCOCME8ssmp8xa9GNt4nQ5ycBJSrqIU5p9q9jXyedKFUO +zgMf+TWJg8MG4OohmOn7nGXVK2ZVGT5yeW3Q41mqg96kGgV78wGWFYtHas7hBnH3 +aTt/oP2XjFY6VYlqwMSh9cqLgqrb9zg5G18Ip0p6wntPHFOFcUrwIjEXGShX+XPn +U9fwPYz1fRaVrflxLDQMZ9WHB4e8d9fW7wuDUkOcL7/8kMcvEvfU+tFGktstKj9w +pZm1IZ0o51L7IwEY6ZwJ4rx73P9e8A4KrbqZwAdDYcArnSZTkvtFS8KYwral4fP1 +6AfWoyaQijINjb+Jr7jPWn6JHeCSaFF7Vrb4wMtxQ5YL0SnTxqS2zFlrgJ0FNIM6 +YebqXVVKvJT+eQTc+LSxEWe73CYCvur9dksty48KpBbaADpwFqi4nUFFxP4L93If +0AlliNEI0YFu4uKh5EmBpr5wbX8KoTBczjTYoVIilerq4yMXSGc= +=ZUVD +-----END PGP SIGNATURE----- diff --git a/share/security/patches/SA-20:21/usb_net.12.patch b/share/security/patches/SA-20:21/usb_net.12.patch new file mode 100644 index 0000000000..1159df905b --- /dev/null +++ b/share/security/patches/SA-20:21/usb_net.12.patch @@ -0,0 +1,113 @@ +--- sys/dev/usb/net/if_cdceem.c.orig ++++ sys/dev/usb/net/if_cdceem.c +@@ -426,9 +426,10 @@ + struct usb_ether *ue; + struct ifnet *ifp; + struct mbuf *m; +- int actlen, off; + uint32_t computed_crc, received_crc; +- uint16_t pktlen; ++ int pktlen; ++ int actlen; ++ int off; + + off = *offp; + sc = usbd_xfer_softc(xfer); +@@ -442,7 +443,7 @@ + (hdr & CDCEEM_DATA_CRC) ? "valid" : "absent", + pktlen); + +- if (pktlen < ETHER_HDR_LEN) { ++ if (pktlen < (ETHER_HDR_LEN + 4)) { + CDCEEM_WARN(sc, + "bad ethernet frame length %d, should be at least %d", + pktlen, ETHER_HDR_LEN); +@@ -466,6 +467,14 @@ + } + + pktlen -= 4; /* Subtract the CRC. */ ++ ++ if (pktlen > m->m_len) { ++ CDCEEM_WARN(sc, "buffer too small %d vs %d bytes", ++ pktlen, m->m_len); ++ if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); ++ m_freem(m); ++ return; ++ } + usbd_copy_out(pc, off, mtod(m, uint8_t *), pktlen); + off += pktlen; + +@@ -512,7 +521,7 @@ + pc = usbd_xfer_get_frame(xfer, 0); + off = 0; + +- while (off < actlen) { ++ while ((off + sizeof(hdr)) <= actlen) { + usbd_copy_out(pc, off, &hdr, sizeof(hdr)); + CDCEEM_DEBUG(sc, "hdr = %#x", hdr); + off += sizeof(hdr); +--- sys/dev/usb/net/if_muge.c.orig ++++ sys/dev/usb/net/if_muge.c +@@ -1166,9 +1166,9 @@ + struct ifnet *ifp = uether_getifp(ue); + struct mbuf *m; + struct usb_page_cache *pc; +- uint16_t pktlen; + uint32_t rx_cmd_a, rx_cmd_b; + uint16_t rx_cmd_c; ++ int pktlen; + int off; + int actlen; + +@@ -1246,7 +1246,14 @@ + 1); + goto tr_setup; + } +- ++ if (pktlen > m->m_len) { ++ muge_dbg_printf(sc, ++ "buffer too small %d vs %d bytes", ++ pktlen, m->m_len); ++ if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); ++ m_freem(m); ++ goto tr_setup; ++ } + usbd_copy_out(pc, off, mtod(m, uint8_t *), + pktlen); + +--- sys/dev/usb/net/if_smsc.c.orig ++++ sys/dev/usb/net/if_smsc.c +@@ -973,7 +973,7 @@ + struct mbuf *m; + struct usb_page_cache *pc; + uint32_t rxhdr; +- uint16_t pktlen; ++ int pktlen; + int off; + int actlen; + +@@ -999,6 +999,9 @@ + /* The frame header is always aligned on a 4 byte boundary */ + off = ((off + 0x3) & ~0x3); + ++ if ((off + sizeof(rxhdr)) > actlen) ++ goto tr_setup; ++ + usbd_copy_out(pc, off, &rxhdr, sizeof(rxhdr)); + off += (sizeof(rxhdr) + ETHER_ALIGN); + rxhdr = le32toh(rxhdr); +@@ -1027,7 +1030,13 @@ + if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); + goto tr_setup; + } +- ++ if (pktlen > m->m_len) { ++ smsc_dbg_printf(sc, "buffer too small %d vs %d bytes", ++ pktlen, m->m_len); ++ if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); ++ m_freem(m); ++ goto tr_setup; ++ } + usbd_copy_out(pc, off, mtod(m, uint8_t *), pktlen); + + /* Check if RX TCP/UDP checksumming is being offloaded */ diff --git a/share/security/patches/SA-20:21/usb_net.12.patch.asc b/share/security/patches/SA-20:21/usb_net.12.patch.asc new file mode 100644 index 0000000000..efd8f15cc9 --- /dev/null +++ b/share/security/patches/SA-20:21/usb_net.12.patch.asc @@ -0,0 +1,18 @@ +-----BEGIN PGP SIGNATURE----- + +iQKTBAABCgB9FiEE/A6HiuWv54gCjWNV05eS9J6n5cIFAl8q63dfFIAAAAAALgAo +aXNzdWVyLWZwckBub3RhdGlvbnMub3BlbnBncC5maWZ0aGhvcnNlbWFuLm5ldEZD +MEU4NzhBRTVBRkU3ODgwMjhENjM1NUQzOTc5MkY0OUVBN0U1QzIACgkQ05eS9J6n +5cK/Rw/9EPrYfgoU/y0Qjp45sx8QyLcTrWasG2pwQT1PJZjes/KviQzy821uTtZG +fJqsAECKUf1Tkr3iKUzfHXFbTEb+oZgm2wbTzeqTSkzEDxd/psRJlA5D6tWk2sN6 +Ws4FzKSzMSM6YVreT0ITm7hwIV1qW1KcN7pjy3YQZtwrK6vYDV5MFB7qHdPe6uLF +hXofqIrw2pyiH7Z6Ok1cIgPqdBVV6t2xpU0daRGSSFIEAoguS2J+tUs8NsjWh1wf +Ihu7o185Z54q4u91vWTYTqBFe1rXmDX6GQgS1qIV3z43woHRy5otedolQBDpPCqz +mcNbITrDkU/ngx9UorJqyD6+hNuo73Px6Qyz0szGlpHeCupAbhCTt/vKj9FGcN+p +q9on5FlGJiZ5KK0JBixzAzTLpjxVyTSEypaOLtTrgNhWcJBL6o0GQqLp+9lv2M9A +x7CfVrrs75H1P44fZnVIr46evxq+I1Si6VCaVtamdQQqz5voawyjhgX7nwsw0gts +BicZVomn+b9EBH2kCIEa20CXIVE76F891JTBBcuasE2+aRI83NTnNfKiBRvizueE +I7BuZfOxm8+lE0A790sA+TbIaNpPZolRWdsV205IKrjDEo0VluJwrdQalUZ7pAxA +IZ07rDKbswFKGLxitlnOL4zWv2xsOKPGh+HcI5gUHGbChTSaH8g= +=pBal +-----END PGP SIGNATURE----- diff --git a/share/security/patches/SA-20:22/sqlite.11.3.patch b/share/security/patches/SA-20:22/sqlite.11.3.patch new file mode 100644 index 0000000000..444249d6ae --- /dev/null +++ b/share/security/patches/SA-20:22/sqlite.11.3.patch @@ -0,0 +1,47236 @@ +--- contrib/sqlite3/Makefile.msc.orig ++++ contrib/sqlite3/Makefile.msc +@@ -73,7 +73,7 @@ + !IFNDEF NO_WARN + !IF $(USE_FULLWARN)!=0 + NO_WARN = -wd4054 -wd4055 -wd4100 -wd4127 -wd4130 -wd4152 -wd4189 -wd4206 +-NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4305 -wd4306 -wd4702 -wd4706 ++NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4244 -wd4305 -wd4306 -wd4702 -wd4706 + !ENDIF + !ENDIF + +@@ -196,6 +196,7 @@ + DEBUG = 0 + !ENDIF + ++ + # Enable use of available compiler optimizations? Normally, this should be + # non-zero. Setting this to zero, thus disabling all compiler optimizations, + # can be useful for testing. +@@ -210,6 +211,12 @@ + SESSION = 0 + !ENDIF + ++# Set this to non-0 to enable support for the rbu extension. ++# ++!IFNDEF RBU ++RBU = 0 ++!ENDIF ++ + # Set the source code file to be used by executables and libraries when + # they need the amalgamation. + # +@@ -282,7 +289,7 @@ + OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1 + OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1 + OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1 +-OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1 ++OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 + OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1 + !ENDIF + OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 +@@ -296,6 +303,13 @@ + OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1 + !ENDIF + ++# Should the rbu extension be enabled? If so, add compilation options ++# to enable it. ++# ++!IF $(RBU)!=0 ++OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RBU=1 ++!ENDIF ++ + # These are the "extended" SQLite compilation options used when compiling for + # the Windows 10 platform. + # +@@ -978,7 +992,7 @@ + sqlite3.def: Replace.exe $(LIBOBJ) + echo EXPORTS > sqlite3.def + dumpbin /all $(LIBOBJ) \ +- | .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup|rebaser)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \ ++ | .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup|rebaser|rbu)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \ + | sort >> sqlite3.def + + $(SQLITE3EXE): shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) +--- contrib/sqlite3/configure.orig ++++ contrib/sqlite3/configure +@@ -1,6 +1,6 @@ + #! /bin/sh + # Guess values for system-dependent variables and create Makefiles. +-# Generated by GNU Autoconf 2.69 for sqlite 3.28.0. ++# Generated by GNU Autoconf 2.69 for sqlite 3.32.2. + # + # Report bugs to <http://www.sqlite.org>. + # +@@ -590,8 +590,8 @@ + # Identity of this package. + PACKAGE_NAME='sqlite' + PACKAGE_TARNAME='sqlite' +-PACKAGE_VERSION='3.28.0' +-PACKAGE_STRING='sqlite 3.28.0' ++PACKAGE_VERSION='3.32.2' ++PACKAGE_STRING='sqlite 3.32.2' + PACKAGE_BUGREPORT='http://www.sqlite.org' + PACKAGE_URL='' + +@@ -1341,7 +1341,7 @@ + # Omit some internal or obsolete options to make the list less imposing. + # This message is too long to be a string in the A/UX 3.1 sh. + cat <<_ACEOF +-\`configure' configures sqlite 3.28.0 to adapt to many kinds of systems. ++\`configure' configures sqlite 3.32.2 to adapt to many kinds of systems. + + Usage: $0 [OPTION]... [VAR=VALUE]... + +@@ -1412,7 +1412,7 @@ + + if test -n "$ac_init_help"; then + case $ac_init_help in +- short | recursive ) echo "Configuration of sqlite 3.28.0:";; ++ short | recursive ) echo "Configuration of sqlite 3.32.2:";; + esac + cat <<\_ACEOF + +@@ -1537,7 +1537,7 @@ + test -n "$ac_init_help" && exit $ac_status + if $ac_init_version; then + cat <<\_ACEOF +-sqlite configure 3.28.0 ++sqlite configure 3.32.2 + generated by GNU Autoconf 2.69 + + Copyright (C) 2012 Free Software Foundation, Inc. +@@ -1952,7 +1952,7 @@ + This file contains any messages produced by compilers while + running configure, to aid debugging if configure makes a mistake. + +-It was created by sqlite $as_me 3.28.0, which was ++It was created by sqlite $as_me 3.32.2, which was + generated by GNU Autoconf 2.69. Invocation command line was + + $ $0 $@ +@@ -2818,7 +2818,7 @@ + + # Define the identity of the package. + PACKAGE='sqlite' +- VERSION='3.28.0' ++ VERSION='3.32.2' + + + cat >>confdefs.h <<_ACEOF +@@ -13653,7 +13653,7 @@ + fi + + if test x"$enable_rtree" = "xyes"; then +- BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_RTREE" ++ BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_GEOPOLY" + fi + #----------------------------------------------------------------------- + +@@ -14438,7 +14438,7 @@ + # report actual input values of CONFIG_FILES etc. instead of their + # values after options handling. + ac_log=" +-This file was extended by sqlite $as_me 3.28.0, which was ++This file was extended by sqlite $as_me 3.32.2, which was + generated by GNU Autoconf 2.69. Invocation command line was + + CONFIG_FILES = $CONFIG_FILES +@@ -14495,7 +14495,7 @@ + cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 + ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" + ac_cs_version="\\ +-sqlite config.status 3.28.0 ++sqlite config.status 3.32.2 + configured by $0, generated by GNU Autoconf 2.69, + with options \\"\$ac_cs_config\\" + +--- contrib/sqlite3/configure.ac.orig ++++ contrib/sqlite3/configure.ac +@@ -10,7 +10,7 @@ + # + + AC_PREREQ(2.61) +-AC_INIT(sqlite, 3.28.0, http://www.sqlite.org) ++AC_INIT(sqlite, 3.32.2, http://www.sqlite.org) + AC_CONFIG_SRCDIR([sqlite3.c]) + AC_CONFIG_AUX_DIR([.]) + +@@ -161,7 +161,7 @@ + [--enable-rtree], [include rtree support [default=yes]])], + [], [enable_rtree=yes]) + if test x"$enable_rtree" = "xyes"; then +- BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_RTREE" ++ BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_GEOPOLY" + fi + #----------------------------------------------------------------------- + +--- contrib/sqlite3/shell.c.orig ++++ contrib/sqlite3/shell.c +@@ -35,6 +35,14 @@ + #define _CRT_SECURE_NO_WARNINGS + #endif + ++/* ++** Determine if we are dealing with WinRT, which provides only a subset of ++** the full Win32 API. ++*/ ++#if !defined(SQLITE_OS_WINRT) ++# define SQLITE_OS_WINRT 0 ++#endif ++ + /* + ** Warning pragmas copied from msvc.h in the core. + */ +@@ -147,22 +155,26 @@ + + + #if defined(_WIN32) || defined(WIN32) +-# include <io.h> +-# include <fcntl.h> +-# define isatty(h) _isatty(h) +-# ifndef access +-# define access(f,m) _access((f),(m)) +-# endif +-# ifndef unlink +-# define unlink _unlink +-# endif +-# ifndef strdup +-# define strdup _strdup ++# if SQLITE_OS_WINRT ++# define SQLITE_OMIT_POPEN 1 ++# else ++# include <io.h> ++# include <fcntl.h> ++# define isatty(h) _isatty(h) ++# ifndef access ++# define access(f,m) _access((f),(m)) ++# endif ++# ifndef unlink ++# define unlink _unlink ++# endif ++# ifndef strdup ++# define strdup _strdup ++# endif ++# undef popen ++# define popen _popen ++# undef pclose ++# define pclose _pclose + # endif +-# undef popen +-# define popen _popen +-# undef pclose +-# define pclose _pclose + #else + /* Make sure isatty() has a prototype. */ + extern int isatty(int); +@@ -191,6 +203,9 @@ + #define ToLower(X) (char)tolower((unsigned char)X) + + #if defined(_WIN32) || defined(WIN32) ++#if SQLITE_OS_WINRT ++#include <intrin.h> ++#endif + #include <windows.h> + + /* string conversion routines only needed on Win32 */ +@@ -206,7 +221,7 @@ + ** rendering quoted strings that contain \n characters). The following + ** routines take care of that. + */ +-#if defined(_WIN32) || defined(WIN32) ++#if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT + static void setBinaryMode(FILE *file, int isOutput){ + if( isOutput ) fflush(file); + _setmode(_fileno(file), _O_BINARY); +@@ -310,6 +325,7 @@ + if( getProcessTimesAddr ){ + return 1; + } else { ++#if !SQLITE_OS_WINRT + /* GetProcessTimes() isn't supported in WIN95 and some other Windows + ** versions. See if the version we are running on has it, and if it + ** does, save off a pointer to it and the current process handle. +@@ -326,6 +342,7 @@ + FreeLibrary(hinstLib); + } + } ++#endif + } + return 0; + } +@@ -415,6 +432,15 @@ + */ + static volatile int seenInterrupt = 0; + ++#ifdef SQLITE_DEBUG ++/* ++** Out-of-memory simulator variables ++*/ ++static unsigned int oomCounter = 0; /* Simulate OOM when equals 1 */ ++static unsigned int oomRepeat = 0; /* Number of OOMs in a row */ ++static void*(*defaultMalloc)(int) = 0; /* The low-level malloc routine */ ++#endif /* SQLITE_DEBUG */ ++ + /* + ** This is the name of our program. It is set in main(), used + ** in a number of other places, mostly for error messages. +@@ -466,6 +492,49 @@ + exit(1); + } + ++#ifdef SQLITE_DEBUG ++/* This routine is called when a simulated OOM occurs. It is broken ++** out as a separate routine to make it easy to set a breakpoint on ++** the OOM ++*/ ++void shellOomFault(void){ ++ if( oomRepeat>0 ){ ++ oomRepeat--; ++ }else{ ++ oomCounter--; ++ } ++} ++#endif /* SQLITE_DEBUG */ ++ ++#ifdef SQLITE_DEBUG ++/* This routine is a replacement malloc() that is used to simulate ++** Out-Of-Memory (OOM) errors for testing purposes. ++*/ ++static void *oomMalloc(int nByte){ ++ if( oomCounter ){ ++ if( oomCounter==1 ){ ++ shellOomFault(); ++ return 0; ++ }else{ ++ oomCounter--; ++ } ++ } ++ return defaultMalloc(nByte); ++} ++#endif /* SQLITE_DEBUG */ ++ ++#ifdef SQLITE_DEBUG ++/* Register the OOM simulator. This must occur before any memory ++** allocations */ ++static void registerOomSimulator(void){ ++ sqlite3_mem_methods mem; ++ sqlite3_config(SQLITE_CONFIG_GETMALLOC, &mem); ++ defaultMalloc = mem.xMalloc; ++ mem.xMalloc = oomMalloc; ++ sqlite3_config(SQLITE_CONFIG_MALLOC, &mem); ++} ++#endif ++ + /* + ** Write I/O traces to the following stream. + */ +@@ -983,6 +1052,7 @@ + ** We need several support functions from the SQLite core. + */ + ++/* #include "sqlite3.h" */ + + /* + ** We need several things from the ANSI and MSVCRT headers. +@@ -1336,6 +1406,7 @@ + ** is used. If SIZE is included it must be one of the integers 224, 256, + ** 384, or 512, to determine SHA3 hash variant that is computed. + */ ++/* #include "sqlite3ext.h" */ + SQLITE_EXTENSION_INIT1 + #include <assert.h> + #include <string.h> +@@ -2005,19 +2076,23 @@ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ +- rc = sqlite3_create_function(db, "sha3", 1, SQLITE_UTF8, 0, +- sha3Func, 0, 0); ++ rc = sqlite3_create_function(db, "sha3", 1, ++ SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, ++ 0, sha3Func, 0, 0); + if( rc==SQLITE_OK ){ +- rc = sqlite3_create_function(db, "sha3", 2, SQLITE_UTF8, 0, +- sha3Func, 0, 0); ++ rc = sqlite3_create_function(db, "sha3", 2, ++ SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, ++ 0, sha3Func, 0, 0); + } + if( rc==SQLITE_OK ){ +- rc = sqlite3_create_function(db, "sha3_query", 1, SQLITE_UTF8, 0, +- sha3QueryFunc, 0, 0); ++ rc = sqlite3_create_function(db, "sha3_query", 1, ++ SQLITE_UTF8 | SQLITE_DIRECTONLY, ++ 0, sha3QueryFunc, 0, 0); + } + if( rc==SQLITE_OK ){ +- rc = sqlite3_create_function(db, "sha3_query", 2, SQLITE_UTF8, 0, +- sha3QueryFunc, 0, 0); ++ rc = sqlite3_create_function(db, "sha3_query", 2, ++ SQLITE_UTF8 | SQLITE_DIRECTONLY, ++ 0, sha3QueryFunc, 0, 0); + } + return rc; + } +@@ -2099,6 +2174,7 @@ + ** And the paths returned in the "name" column of the table are also + ** relative to directory $dir. + */ ++/* #include "sqlite3ext.h" */ + SQLITE_EXTENSION_INIT1 + #include <stdio.h> + #include <string.h> +@@ -2419,6 +2495,7 @@ + + if( mtime>=0 ){ + #if defined(_WIN32) ++#if !SQLITE_OS_WINRT + /* Windows */ + FILETIME lastAccess; + FILETIME lastWrite; +@@ -2449,6 +2526,7 @@ + }else{ + return 1; + } ++#endif + #elif defined(AT_FDCWD) && 0 /* utimensat() is not universally available */ + /* Recent unix */ + struct timespec times[2]; +@@ -2610,6 +2688,7 @@ + pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); ++ sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); + } + *ppVtab = (sqlite3_vtab*)pNew; + return rc; +@@ -3003,10 +3082,12 @@ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ +- rc = sqlite3_create_function(db, "readfile", 1, SQLITE_UTF8, 0, ++ rc = sqlite3_create_function(db, "readfile", 1, ++ SQLITE_UTF8|SQLITE_DIRECTONLY, 0, + readfileFunc, 0, 0); + if( rc==SQLITE_OK ){ +- rc = sqlite3_create_function(db, "writefile", -1, SQLITE_UTF8, 0, ++ rc = sqlite3_create_function(db, "writefile", -1, ++ SQLITE_UTF8|SQLITE_DIRECTONLY, 0, + writefileFunc, 0, 0); + } + if( rc==SQLITE_OK ){ +@@ -3056,6 +3137,7 @@ + ** faster than any human can type. + ** + */ ++/* #include "sqlite3ext.h" */ + SQLITE_EXTENSION_INIT1 + #include <assert.h> + #include <string.h> +@@ -3140,6 +3222,7 @@ + #define COMPLETION_COLUMN_WHOLELINE 2 /* Entire line seen so far */ + #define COMPLETION_COLUMN_PHASE 3 /* ePhase - used for debugging only */ + ++ sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(" + " candidate TEXT," +@@ -3572,6 +3655,7 @@ + ** If the file being opened is not an appended database, then this shim is + ** a pass-through into the default underlying VFS. + **/ ++/* #include "sqlite3ext.h" */ + SQLITE_EXTENSION_INIT1 + #include <string.h> + #include <assert.h> +@@ -4200,6 +4284,101 @@ + } + + /************************* End ../ext/misc/memtrace.c ********************/ ++/************************* Begin ../ext/misc/uint.c ******************/ ++/* ++** 2020-04-14 ++** ++** The author disclaims copyright to this source code. In place of ++** a legal notice, here is a blessing: ++** ++** May you do good and not evil. ++** May you find forgiveness for yourself and forgive others. ++** May you share freely, never taking more than you give. ++** ++****************************************************************************** ++** ++** This SQLite extension implements the UINT collating sequence. ++** ++** UINT works like BINARY for text, except that embedded strings ++** of digits compare in numeric order. ++** ++** * Leading zeros are handled properly, in the sense that ++** they do not mess of the maginitude comparison of embedded ++** strings of digits. "x00123y" is equal to "x123y". ++** ++** * Only unsigned integers are recognized. Plus and minus ++** signs are ignored. Decimal points and exponential notation ++** are ignored. ++** ++** * Embedded integers can be of arbitrary length. Comparison ++** is *not* limited integers that can be expressed as a ++** 64-bit machine integer. ++*/ ++/* #include "sqlite3ext.h" */ ++SQLITE_EXTENSION_INIT1 ++#include <assert.h> ++#include <string.h> ++#include <ctype.h> ++ ++/* ++** Compare text in lexicographic order, except strings of digits ++** compare in numeric order. ++*/ ++static int uintCollFunc( ++ void *notUsed, ++ int nKey1, const void *pKey1, ++ int nKey2, const void *pKey2 ++){ ++ const unsigned char *zA = (const unsigned char*)pKey1; ++ const unsigned char *zB = (const unsigned char*)pKey2; ++ int i=0, j=0, x; ++ (void)notUsed; ++ while( i<nKey1 && j<nKey2 ){ ++ x = zA[i] - zB[j]; ++ if( isdigit(zA[i]) ){ ++ int k; ++ if( !isdigit(zB[j]) ) return x; ++ while( i<nKey1 && zA[i]=='0' ){ i++; } ++ while( j<nKey2 && zB[j]=='0' ){ j++; } ++ k = 0; ++ while( i+k<nKey1 && isdigit(zA[i+k]) ++ && j+k<nKey2 && isdigit(zB[j+k]) ){ ++ k++; ++ } ++ if( i+k<nKey1 && isdigit(zA[i+k]) ){ ++ return +1; ++ }else if( j+k<nKey2 && isdigit(zB[j+k]) ){ ++ return -1; ++ }else{ ++ x = memcmp(zA+i, zB+j, k); ++ if( x ) return x; ++ i += k; ++ j += k; ++ } ++ }else if( x ){ ++ return x; ++ }else{ ++ i++; ++ j++; ++ } ++ } ++ return (nKey1 - i) - (nKey2 - j); ++} ++ ++#ifdef _WIN32 ++ ++#endif ++int sqlite3_uint_init( ++ sqlite3 *db, ++ char **pzErrMsg, ++ const sqlite3_api_routines *pApi ++){ ++ SQLITE_EXTENSION_INIT2(pApi); ++ (void)pzErrMsg; /* Unused parameter */ ++ return sqlite3_create_collation(db, "uint", SQLITE_UTF8, 0, uintCollFunc); ++} ++ ++/************************* End ../ext/misc/uint.c ********************/ + #ifdef SQLITE_HAVE_ZLIB + /************************* Begin ../ext/misc/zipfile.c ******************/ + /* +@@ -4228,6 +4407,7 @@ + ** * No support for zip64 extensions + ** * Only the "inflate/deflate" (zlib) compression method is supported + */ ++/* #include "sqlite3ext.h" */ + SQLITE_EXTENSION_INIT1 + #include <stdio.h> + #include <string.h> +@@ -4572,6 +4752,7 @@ + zipfileDequote(pNew->zFile); + } + } ++ sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); + *ppVtab = (sqlite3_vtab*)pNew; + return rc; + } +@@ -5184,25 +5365,25 @@ + u8 **ppOut, int *pnOut, /* Output */ + char **pzErr /* OUT: Error message */ + ){ +- sqlite3_int64 nAlloc = compressBound(nIn); +- u8 *aOut; + int rc = SQLITE_OK; ++ sqlite3_int64 nAlloc; ++ z_stream str; ++ u8 *aOut; ++ ++ memset(&str, 0, sizeof(str)); ++ str.next_in = (Bytef*)aIn; ++ str.avail_in = nIn; ++ deflateInit2(&str, 9, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); + ++ nAlloc = deflateBound(&str, nIn); + aOut = (u8*)sqlite3_malloc64(nAlloc); + if( aOut==0 ){ + rc = SQLITE_NOMEM; + }else{ + int res; +- z_stream str; +- memset(&str, 0, sizeof(str)); +- str.next_in = (Bytef*)aIn; +- str.avail_in = nIn; + str.next_out = aOut; + str.avail_out = nAlloc; +- +- deflateInit2(&str, 9, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); + res = deflate(&str, Z_FINISH); +- + if( res==Z_STREAM_END ){ + *ppOut = aOut; + *pnOut = (int)str.total_out; +@@ -5511,10 +5692,10 @@ + idx = i; + } + } ++ pIdxInfo->estimatedCost = 1000.0; + if( idx>=0 ){ + pIdxInfo->aConstraintUsage[idx].argvIndex = 1; + pIdxInfo->aConstraintUsage[idx].omit = 1; +- pIdxInfo->estimatedCost = 1000.0; + pIdxInfo->idxNum = 1; + }else if( unusable ){ + return SQLITE_CONSTRAINT; +@@ -5636,8 +5817,8 @@ + ** identical, ignoring any trailing '/' character in either path. */ + static int zipfileComparePath(const char *zA, const char *zB, int nB){ + int nA = (int)strlen(zA); +- if( zA[nA-1]=='/' ) nA--; +- if( zB[nB-1]=='/' ) nB--; ++ if( nA>0 && zA[nA-1]=='/' ) nA--; ++ if( nB>0 && zB[nB-1]=='/' ) nB--; + if( nA==nB && memcmp(zA, zB, nA)==0 ) return 0; + return 1; + } +@@ -5647,6 +5828,10 @@ + int rc = SQLITE_OK; + + assert( pTab->pWriteFd==0 ); ++ if( pTab->zFile==0 || pTab->zFile[0]==0 ){ ++ pTab->base.zErrMsg = sqlite3_mprintf("zipfile: missing filename"); ++ return SQLITE_ERROR; ++ } + + /* Open a write fd on the file. Also load the entire central directory + ** structure into memory. During the transaction any new file data is +@@ -5821,6 +6006,7 @@ + + if( rc==SQLITE_OK ){ + zPath = (const char*)sqlite3_value_text(apVal[2]); ++ if( zPath==0 ) zPath = ""; + nPath = (int)strlen(zPath); + mTime = zipfileGetTime(apVal[4]); + } +@@ -5830,11 +6016,15 @@ + ** '/'. This appears to be required for compatibility with info-zip + ** (the unzip command on unix). It does not create directories + ** otherwise. */ +- if( zPath[nPath-1]!='/' ){ ++ if( nPath<=0 || zPath[nPath-1]!='/' ){ + zFree = sqlite3_mprintf("%s/", zPath); +- if( zFree==0 ){ rc = SQLITE_NOMEM; } + zPath = (const char*)zFree; +- nPath++; ++ if( zFree==0 ){ ++ rc = SQLITE_NOMEM; ++ nPath = 0; ++ }else{ ++ nPath = (int)strlen(zPath); ++ } + } + } + +@@ -6227,19 +6417,19 @@ + ** at the end of the path. Or, if this is not a directory and the path + ** ends in '/' it is an error. */ + if( bIsDir==0 ){ +- if( zName[nName-1]=='/' ){ ++ if( nName>0 && zName[nName-1]=='/' ){ + zErr = sqlite3_mprintf("non-directory name must not end with /"); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + }else{ +- if( zName[nName-1]!='/' ){ ++ if( nName==0 || zName[nName-1]!='/' ){ + zName = zFree = sqlite3_mprintf("%s/", zName); +- nName++; + if( zName==0 ){ + rc = SQLITE_NOMEM; + goto zipfile_step_out; + } ++ nName = (int)strlen(zName); + }else{ + while( nName>1 && zName[nName-2]=='/' ) nName--; + } +@@ -6398,8 +6588,10 @@ + ** for working with sqlar archives and used by the shell tool's built-in + ** sqlar support. + */ ++/* #include "sqlite3ext.h" */ + SQLITE_EXTENSION_INIT1 + #include <zlib.h> ++#include <assert.h> + + /* + ** Implementation of the "sqlar_compress(X)" SQL function. +@@ -6494,10 +6686,12 @@ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ +- rc = sqlite3_create_function(db, "sqlar_compress", 1, SQLITE_UTF8, 0, ++ rc = sqlite3_create_function(db, "sqlar_compress", 1, ++ SQLITE_UTF8|SQLITE_INNOCUOUS, 0, + sqlarCompressFunc, 0, 0); + if( rc==SQLITE_OK ){ +- rc = sqlite3_create_function(db, "sqlar_uncompress", 2, SQLITE_UTF8, 0, ++ rc = sqlite3_create_function(db, "sqlar_uncompress", 2, ++ SQLITE_UTF8|SQLITE_INNOCUOUS, 0, + sqlarUncompressFunc, 0, 0); + } + return rc; +@@ -6518,8 +6712,9 @@ + ** + ************************************************************************* + */ +- +- ++#if !defined(SQLITEEXPERT_H) ++#define SQLITEEXPERT_H 1 ++/* #include "sqlite3.h" */ + + typedef struct sqlite3expert sqlite3expert; + +@@ -6672,7 +6867,7 @@ + */ + void sqlite3_expert_destroy(sqlite3expert*); + +- ++#endif /* !defined(SQLITEEXPERT_H) */ + + /************************* End ../ext/expert/sqlite3expert.h ********************/ + /************************* Begin ../ext/expert/sqlite3expert.c ******************/ +@@ -6688,6 +6883,7 @@ + ** + ************************************************************************* + */ ++/* #include "sqlite3expert.h" */ + #include <assert.h> + #include <string.h> + #include <stdio.h> +@@ -7805,14 +8001,19 @@ + /* int iParent = sqlite3_column_int(pExplain, 1); */ + /* int iNotUsed = sqlite3_column_int(pExplain, 2); */ + const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3); +- int nDetail = STRLEN(zDetail); ++ int nDetail; + int i; + ++ if( !zDetail ) continue; ++ nDetail = STRLEN(zDetail); ++ + for(i=0; i<nDetail; i++){ + const char *zIdx = 0; +- if( memcmp(&zDetail[i], " USING INDEX ", 13)==0 ){ ++ if( i+13<nDetail && memcmp(&zDetail[i], " USING INDEX ", 13)==0 ){ + zIdx = &zDetail[i+13]; +- }else if( memcmp(&zDetail[i], " USING COVERING INDEX ", 22)==0 ){ ++ }else if( i+22<nDetail ++ && memcmp(&zDetail[i], " USING COVERING INDEX ", 22)==0 ++ ){ + zIdx = &zDetail[i+22]; + } + if( zIdx ){ +@@ -8627,129 +8828,988 @@ + } + } + +-#endif /* ifndef SQLITE_OMIT_VIRTUAL_TABLE */ ++#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + + /************************* End ../ext/expert/sqlite3expert.c ********************/ + +-#if defined(SQLITE_ENABLE_SESSION) ++#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) ++/************************* Begin ../ext/misc/dbdata.c ******************/ + /* +-** State information for a single open session ++** 2019-04-17 ++** ++** The author disclaims copyright to this source code. In place of ++** a legal notice, here is a blessing: ++** ++** May you do good and not evil. ++** May you find forgiveness for yourself and forgive others. ++** May you share freely, never taking more than you give. ++** ++****************************************************************************** ++** ++** This file contains an implementation of two eponymous virtual tables, ++** "sqlite_dbdata" and "sqlite_dbptr". Both modules require that the ++** "sqlite_dbpage" eponymous virtual table be available. ++** ++** SQLITE_DBDATA: ++** sqlite_dbdata is used to extract data directly from a database b-tree ++** page and its associated overflow pages, bypassing the b-tree layer. ++** The table schema is equivalent to: ++** ++** CREATE TABLE sqlite_dbdata( ++** pgno INTEGER, ++** cell INTEGER, ++** field INTEGER, ++** value ANY, ++** schema TEXT HIDDEN ++** ); ++** ++** IMPORTANT: THE VIRTUAL TABLE SCHEMA ABOVE IS SUBJECT TO CHANGE. IN THE ++** FUTURE NEW NON-HIDDEN COLUMNS MAY BE ADDED BETWEEN "value" AND ++** "schema". ++** ++** Each page of the database is inspected. If it cannot be interpreted as ++** a b-tree page, or if it is a b-tree page containing 0 entries, the ++** sqlite_dbdata table contains no rows for that page. Otherwise, the ++** table contains one row for each field in the record associated with ++** each cell on the page. For intkey b-trees, the key value is stored in ++** field -1. ++** ++** For example, for the database: ++** ++** CREATE TABLE t1(a, b); -- root page is page 2 ++** INSERT INTO t1(rowid, a, b) VALUES(5, 'v', 'five'); ++** INSERT INTO t1(rowid, a, b) VALUES(10, 'x', 'ten'); ++** ++** the sqlite_dbdata table contains, as well as from entries related to ++** page 1, content equivalent to: ++** ++** INSERT INTO sqlite_dbdata(pgno, cell, field, value) VALUES ++** (2, 0, -1, 5 ), ++** (2, 0, 0, 'v' ), ++** (2, 0, 1, 'five'), ++** (2, 1, -1, 10 ), ++** (2, 1, 0, 'x' ), ++** (2, 1, 1, 'ten' ); ++** ++** If database corruption is encountered, this module does not report an ++** error. Instead, it attempts to extract as much data as possible and ++** ignores the corruption. ++** ++** SQLITE_DBPTR: ++** The sqlite_dbptr table has the following schema: ++** ++** CREATE TABLE sqlite_dbptr( ++** pgno INTEGER, ++** child INTEGER, ++** schema TEXT HIDDEN ++** ); ++** ++** It contains one entry for each b-tree pointer between a parent and ++** child page in the database. + */ +-typedef struct OpenSession OpenSession; +-struct OpenSession { +- char *zName; /* Symbolic name for this session */ +- int nFilter; /* Number of xFilter rejection GLOB patterns */ +- char **azFilter; /* Array of xFilter rejection GLOB patterns */ +- sqlite3_session *p; /* The open session */ +-}; ++#if !defined(SQLITEINT_H) ++/* #include "sqlite3ext.h" */ ++ ++/* typedef unsigned char u8; */ ++ + #endif ++SQLITE_EXTENSION_INIT1 ++#include <string.h> ++#include <assert.h> + +-/* +-** Shell output mode information from before ".explain on", +-** saved so that it can be restored by ".explain off" +-*/ +-typedef struct SavedModeInfo SavedModeInfo; +-struct SavedModeInfo { +- int valid; /* Is there legit data in here? */ +- int mode; /* Mode prior to ".explain on" */ +- int showHeader; /* The ".header" setting prior to ".explain on" */ +- int colWidth[100]; /* Column widths prior to ".explain on" */ ++#define DBDATA_PADDING_BYTES 100 ++ ++typedef struct DbdataTable DbdataTable; ++typedef struct DbdataCursor DbdataCursor; ++ ++/* Cursor object */ ++struct DbdataCursor { ++ sqlite3_vtab_cursor base; /* Base class. Must be first */ ++ sqlite3_stmt *pStmt; /* For fetching database pages */ ++ ++ int iPgno; /* Current page number */ ++ u8 *aPage; /* Buffer containing page */ ++ int nPage; /* Size of aPage[] in bytes */ ++ int nCell; /* Number of cells on aPage[] */ ++ int iCell; /* Current cell number */ ++ int bOnePage; /* True to stop after one page */ ++ int szDb; ++ sqlite3_int64 iRowid; ++ ++ /* Only for the sqlite_dbdata table */ ++ u8 *pRec; /* Buffer containing current record */ ++ int nRec; /* Size of pRec[] in bytes */ ++ int nHdr; /* Size of header in bytes */ ++ int iField; /* Current field number */ ++ u8 *pHdrPtr; ++ u8 *pPtr; ++ ++ sqlite3_int64 iIntkey; /* Integer key value */ + }; + +-typedef struct ExpertInfo ExpertInfo; +-struct ExpertInfo { +- sqlite3expert *pExpert; +- int bVerbose; ++/* Table object */ ++struct DbdataTable { ++ sqlite3_vtab base; /* Base class. Must be first */ ++ sqlite3 *db; /* The database connection */ ++ sqlite3_stmt *pStmt; /* For fetching database pages */ ++ int bPtr; /* True for sqlite3_dbptr table */ + }; + +-/* A single line in the EQP output */ +-typedef struct EQPGraphRow EQPGraphRow; +-struct EQPGraphRow { +- int iEqpId; /* ID for this row */ +- int iParentId; /* ID of the parent row */ +- EQPGraphRow *pNext; /* Next row in sequence */ +- char zText[1]; /* Text to display for this row */ +-}; ++/* Column and schema definitions for sqlite_dbdata */ ++#define DBDATA_COLUMN_PGNO 0 ++#define DBDATA_COLUMN_CELL 1 ++#define DBDATA_COLUMN_FIELD 2 ++#define DBDATA_COLUMN_VALUE 3 ++#define DBDATA_COLUMN_SCHEMA 4 ++#define DBDATA_SCHEMA \ ++ "CREATE TABLE x(" \ ++ " pgno INTEGER," \ ++ " cell INTEGER," \ ++ " field INTEGER," \ ++ " value ANY," \ ++ " schema TEXT HIDDEN" \ ++ ")" ++ ++/* Column and schema definitions for sqlite_dbptr */ ++#define DBPTR_COLUMN_PGNO 0 ++#define DBPTR_COLUMN_CHILD 1 ++#define DBPTR_COLUMN_SCHEMA 2 ++#define DBPTR_SCHEMA \ ++ "CREATE TABLE x(" \ ++ " pgno INTEGER," \ ++ " child INTEGER," \ ++ " schema TEXT HIDDEN" \ ++ ")" ++ ++/* ++** Connect to an sqlite_dbdata (pAux==0) or sqlite_dbptr (pAux!=0) virtual ++** table. ++*/ ++static int dbdataConnect( ++ sqlite3 *db, ++ void *pAux, ++ int argc, const char *const*argv, ++ sqlite3_vtab **ppVtab, ++ char **pzErr ++){ ++ DbdataTable *pTab = 0; ++ int rc = sqlite3_declare_vtab(db, pAux ? DBPTR_SCHEMA : DBDATA_SCHEMA); + +-/* All EQP output is collected into an instance of the following */ +-typedef struct EQPGraph EQPGraph; +-struct EQPGraph { +- EQPGraphRow *pRow; /* Linked list of all rows of the EQP output */ +- EQPGraphRow *pLast; /* Last element of the pRow list */ +- char zPrefix[100]; /* Graph prefix */ +-}; ++ if( rc==SQLITE_OK ){ ++ pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable)); ++ if( pTab==0 ){ ++ rc = SQLITE_NOMEM; ++ }else{ ++ memset(pTab, 0, sizeof(DbdataTable)); ++ pTab->db = db; ++ pTab->bPtr = (pAux!=0); ++ } ++ } ++ ++ *ppVtab = (sqlite3_vtab*)pTab; ++ return rc; ++} + + /* +-** State information about the database connection is contained in an +-** instance of the following structure. ++** Disconnect from or destroy a sqlite_dbdata or sqlite_dbptr virtual table. + */ +-typedef struct ShellState ShellState; +-struct ShellState { +- sqlite3 *db; /* The database */ +- u8 autoExplain; /* Automatically turn on .explain mode */ +- u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */ +- u8 autoEQPtest; /* autoEQP is in test mode */ +- u8 autoEQPtrace; /* autoEQP is in trace mode */ +- u8 statsOn; /* True to display memory stats before each finalize */ +- u8 scanstatsOn; /* True to display scan stats before each finalize */ +- u8 openMode; /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */ +- u8 doXdgOpen; /* Invoke start/open/xdg-open in output_reset() */ +- u8 nEqpLevel; /* Depth of the EQP output graph */ +- u8 eTraceType; /* SHELL_TRACE_* value for type of trace */ +- unsigned mEqpLines; /* Mask of veritical lines in the EQP output graph */ +- int outCount; /* Revert to stdout when reaching zero */ +- int cnt; /* Number of records displayed so far */ +- int lineno; /* Line number of last line read from in */ +- FILE *in; /* Read commands from this stream */ +- FILE *out; /* Write results here */ +- FILE *traceOut; /* Output for sqlite3_trace() */ +- int nErr; /* Number of errors seen */ +- int mode; /* An output mode setting */ +- int modePrior; /* Saved mode */ +- int cMode; /* temporary output mode for the current query */ +- int normalMode; /* Output mode before ".explain on" */ +- int writableSchema; /* True if PRAGMA writable_schema=ON */ +- int showHeader; /* True to show column names in List or Column mode */ +- int nCheck; /* Number of ".check" commands run */ +- unsigned nProgress; /* Number of progress callbacks encountered */ +- unsigned mxProgress; /* Maximum progress callbacks before failing */ +- unsigned flgProgress; /* Flags for the progress callback */ +- unsigned shellFlgs; /* Various flags */ +- sqlite3_int64 szMax; /* --maxsize argument to .open */ +- char *zDestTable; /* Name of destination table when MODE_Insert */ +- char *zTempFile; /* Temporary file that might need deleting */ +- char zTestcase[30]; /* Name of current test case */ +- char colSeparator[20]; /* Column separator character for several modes */ +- char rowSeparator[20]; /* Row separator character for MODE_Ascii */ +- char colSepPrior[20]; /* Saved column separator */ +- char rowSepPrior[20]; /* Saved row separator */ +- int colWidth[100]; /* Requested width of each column when in column mode*/ +- int actualWidth[100]; /* Actual width of each column */ +- char nullValue[20]; /* The text to print when a NULL comes back from +- ** the database */ +- char outfile[FILENAME_MAX]; /* Filename for *out */ +- const char *zDbFilename; /* name of the database file */ +- char *zFreeOnClose; /* Filename to free when closing */ +- const char *zVfs; /* Name of VFS to use */ +- sqlite3_stmt *pStmt; /* Current statement if any. */ +- FILE *pLog; /* Write log output here */ +- int *aiIndent; /* Array of indents used in MODE_Explain */ +- int nIndent; /* Size of array aiIndent[] */ +- int iIndent; /* Index of current op in aiIndent[] */ +- EQPGraph sGraph; /* Information for the graphical EXPLAIN QUERY PLAN */ +-#if defined(SQLITE_ENABLE_SESSION) +- int nSession; /* Number of active sessions */ +- OpenSession aSession[4]; /* Array of sessions. [0] is in focus. */ +-#endif +- ExpertInfo expert; /* Valid if previous command was ".expert OPT..." */ +-}; +- ++static int dbdataDisconnect(sqlite3_vtab *pVtab){ ++ DbdataTable *pTab = (DbdataTable*)pVtab; ++ if( pTab ){ ++ sqlite3_finalize(pTab->pStmt); ++ sqlite3_free(pVtab); ++ } ++ return SQLITE_OK; ++} + +-/* Allowed values for ShellState.autoEQP ++/* ++** This function interprets two types of constraints: ++** ++** schema=? ++** pgno=? ++** ++** If neither are present, idxNum is set to 0. If schema=? is present, ++** the 0x01 bit in idxNum is set. If pgno=? is present, the 0x02 bit ++** in idxNum is set. ++** ++** If both parameters are present, schema is in position 0 and pgno in ++** position 1. + */ +-#define AUTOEQP_off 0 /* Automatic EXPLAIN QUERY PLAN is off */ +-#define AUTOEQP_on 1 /* Automatic EQP is on */ +-#define AUTOEQP_trigger 2 /* On and also show plans for triggers */ ++static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdx){ ++ DbdataTable *pTab = (DbdataTable*)tab; ++ int i; ++ int iSchema = -1; ++ int iPgno = -1; ++ int colSchema = (pTab->bPtr ? DBPTR_COLUMN_SCHEMA : DBDATA_COLUMN_SCHEMA); ++ ++ for(i=0; i<pIdx->nConstraint; i++){ ++ struct sqlite3_index_constraint *p = &pIdx->aConstraint[i]; ++ if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ ++ if( p->iColumn==colSchema ){ ++ if( p->usable==0 ) return SQLITE_CONSTRAINT; ++ iSchema = i; ++ } ++ if( p->iColumn==DBDATA_COLUMN_PGNO && p->usable ){ ++ iPgno = i; ++ } ++ } ++ } ++ ++ if( iSchema>=0 ){ ++ pIdx->aConstraintUsage[iSchema].argvIndex = 1; ++ pIdx->aConstraintUsage[iSchema].omit = 1; ++ } ++ if( iPgno>=0 ){ ++ pIdx->aConstraintUsage[iPgno].argvIndex = 1 + (iSchema>=0); ++ pIdx->aConstraintUsage[iPgno].omit = 1; ++ pIdx->estimatedCost = 100; ++ pIdx->estimatedRows = 50; ++ ++ if( pTab->bPtr==0 && pIdx->nOrderBy && pIdx->aOrderBy[0].desc==0 ){ ++ int iCol = pIdx->aOrderBy[0].iColumn; ++ if( pIdx->nOrderBy==1 ){ ++ pIdx->orderByConsumed = (iCol==0 || iCol==1); ++ }else if( pIdx->nOrderBy==2 && pIdx->aOrderBy[1].desc==0 && iCol==0 ){ ++ pIdx->orderByConsumed = (pIdx->aOrderBy[1].iColumn==1); ++ } ++ } ++ ++ }else{ ++ pIdx->estimatedCost = 100000000; ++ pIdx->estimatedRows = 1000000000; ++ } ++ pIdx->idxNum = (iSchema>=0 ? 0x01 : 0x00) | (iPgno>=0 ? 0x02 : 0x00); ++ return SQLITE_OK; ++} ++ ++/* ++** Open a new sqlite_dbdata or sqlite_dbptr cursor. ++*/ ++static int dbdataOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ ++ DbdataCursor *pCsr; ++ ++ pCsr = (DbdataCursor*)sqlite3_malloc64(sizeof(DbdataCursor)); ++ if( pCsr==0 ){ ++ return SQLITE_NOMEM; ++ }else{ ++ memset(pCsr, 0, sizeof(DbdataCursor)); ++ pCsr->base.pVtab = pVTab; ++ } ++ ++ *ppCursor = (sqlite3_vtab_cursor *)pCsr; ++ return SQLITE_OK; ++} ++ ++/* ++** Restore a cursor object to the state it was in when first allocated ++** by dbdataOpen(). ++*/ ++static void dbdataResetCursor(DbdataCursor *pCsr){ ++ DbdataTable *pTab = (DbdataTable*)(pCsr->base.pVtab); ++ if( pTab->pStmt==0 ){ ++ pTab->pStmt = pCsr->pStmt; ++ }else{ ++ sqlite3_finalize(pCsr->pStmt); ++ } ++ pCsr->pStmt = 0; ++ pCsr->iPgno = 1; ++ pCsr->iCell = 0; ++ pCsr->iField = 0; ++ pCsr->bOnePage = 0; ++ sqlite3_free(pCsr->aPage); ++ sqlite3_free(pCsr->pRec); ++ pCsr->pRec = 0; ++ pCsr->aPage = 0; ++} ++ ++/* ++** Close an sqlite_dbdata or sqlite_dbptr cursor. ++*/ ++static int dbdataClose(sqlite3_vtab_cursor *pCursor){ ++ DbdataCursor *pCsr = (DbdataCursor*)pCursor; ++ dbdataResetCursor(pCsr); ++ sqlite3_free(pCsr); ++ return SQLITE_OK; ++} ++ ++/* ++** Utility methods to decode 16 and 32-bit big-endian unsigned integers. ++*/ ++static unsigned int get_uint16(unsigned char *a){ ++ return (a[0]<<8)|a[1]; ++} ++static unsigned int get_uint32(unsigned char *a){ ++ return ((unsigned int)a[0]<<24) ++ | ((unsigned int)a[1]<<16) ++ | ((unsigned int)a[2]<<8) ++ | ((unsigned int)a[3]); ++} ++ ++/* ++** Load page pgno from the database via the sqlite_dbpage virtual table. ++** If successful, set (*ppPage) to point to a buffer containing the page ++** data, (*pnPage) to the size of that buffer in bytes and return ++** SQLITE_OK. In this case it is the responsibility of the caller to ++** eventually free the buffer using sqlite3_free(). ++** ++** Or, if an error occurs, set both (*ppPage) and (*pnPage) to 0 and ++** return an SQLite error code. ++*/ ++static int dbdataLoadPage( ++ DbdataCursor *pCsr, /* Cursor object */ ++ unsigned int pgno, /* Page number of page to load */ ++ u8 **ppPage, /* OUT: pointer to page buffer */ ++ int *pnPage /* OUT: Size of (*ppPage) in bytes */ ++){ ++ int rc2; ++ int rc = SQLITE_OK; ++ sqlite3_stmt *pStmt = pCsr->pStmt; ++ ++ *ppPage = 0; ++ *pnPage = 0; ++ sqlite3_bind_int64(pStmt, 2, pgno); ++ if( SQLITE_ROW==sqlite3_step(pStmt) ){ ++ int nCopy = sqlite3_column_bytes(pStmt, 0); ++ if( nCopy>0 ){ ++ u8 *pPage; ++ pPage = (u8*)sqlite3_malloc64(nCopy + DBDATA_PADDING_BYTES); ++ if( pPage==0 ){ ++ rc = SQLITE_NOMEM; ++ }else{ ++ const u8 *pCopy = sqlite3_column_blob(pStmt, 0); ++ memcpy(pPage, pCopy, nCopy); ++ memset(&pPage[nCopy], 0, DBDATA_PADDING_BYTES); ++ } ++ *ppPage = pPage; ++ *pnPage = nCopy; ++ } ++ } ++ rc2 = sqlite3_reset(pStmt); ++ if( rc==SQLITE_OK ) rc = rc2; ++ ++ return rc; ++} ++ ++/* ++** Read a varint. Put the value in *pVal and return the number of bytes. ++*/ ++static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){ ++ sqlite3_int64 v = 0; ++ int i; ++ for(i=0; i<8; i++){ ++ v = (v<<7) + (z[i]&0x7f); ++ if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; } ++ } ++ v = (v<<8) + (z[i]&0xff); ++ *pVal = v; ++ return 9; ++} ++ ++/* ++** Return the number of bytes of space used by an SQLite value of type ++** eType. ++*/ ++static int dbdataValueBytes(int eType){ ++ switch( eType ){ ++ case 0: case 8: case 9: ++ case 10: case 11: ++ return 0; ++ case 1: ++ return 1; ++ case 2: ++ return 2; ++ case 3: ++ return 3; ++ case 4: ++ return 4; ++ case 5: ++ return 6; ++ case 6: ++ case 7: ++ return 8; ++ default: ++ if( eType>0 ){ ++ return ((eType-12) / 2); ++ } ++ return 0; ++ } ++} ++ ++/* ++** Load a value of type eType from buffer pData and use it to set the ++** result of context object pCtx. ++*/ ++static void dbdataValue( ++ sqlite3_context *pCtx, ++ int eType, ++ u8 *pData, ++ int nData ++){ ++ if( eType>=0 && dbdataValueBytes(eType)<=nData ){ ++ switch( eType ){ ++ case 0: ++ case 10: ++ case 11: ++ sqlite3_result_null(pCtx); ++ break; ++ ++ case 8: ++ sqlite3_result_int(pCtx, 0); ++ break; ++ case 9: ++ sqlite3_result_int(pCtx, 1); ++ break; ++ ++ case 1: case 2: case 3: case 4: case 5: case 6: case 7: { ++ sqlite3_uint64 v = (signed char)pData[0]; ++ pData++; ++ switch( eType ){ ++ case 7: ++ case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; ++ case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; ++ case 4: v = (v<<8) + pData[0]; pData++; ++ case 3: v = (v<<8) + pData[0]; pData++; ++ case 2: v = (v<<8) + pData[0]; pData++; ++ } ++ ++ if( eType==7 ){ ++ double r; ++ memcpy(&r, &v, sizeof(r)); ++ sqlite3_result_double(pCtx, r); ++ }else{ ++ sqlite3_result_int64(pCtx, (sqlite3_int64)v); ++ } ++ break; ++ } ++ ++ default: { ++ int n = ((eType-12) / 2); ++ if( eType % 2 ){ ++ sqlite3_result_text(pCtx, (const char*)pData, n, SQLITE_TRANSIENT); ++ }else{ ++ sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); ++ } ++ } ++ } ++ } ++} ++ ++/* ++** Move an sqlite_dbdata or sqlite_dbptr cursor to the next entry. ++*/ ++static int dbdataNext(sqlite3_vtab_cursor *pCursor){ ++ DbdataCursor *pCsr = (DbdataCursor*)pCursor; ++ DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; ++ ++ pCsr->iRowid++; ++ while( 1 ){ ++ int rc; ++ int iOff = (pCsr->iPgno==1 ? 100 : 0); ++ int bNextPage = 0; ++ ++ if( pCsr->aPage==0 ){ ++ while( 1 ){ ++ if( pCsr->bOnePage==0 && pCsr->iPgno>pCsr->szDb ) return SQLITE_OK; ++ rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); ++ if( rc!=SQLITE_OK ) return rc; ++ if( pCsr->aPage ) break; ++ pCsr->iPgno++; ++ } ++ pCsr->iCell = pTab->bPtr ? -2 : 0; ++ pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); ++ } ++ ++ if( pTab->bPtr ){ ++ if( pCsr->aPage[iOff]!=0x02 && pCsr->aPage[iOff]!=0x05 ){ ++ pCsr->iCell = pCsr->nCell; ++ } ++ pCsr->iCell++; ++ if( pCsr->iCell>=pCsr->nCell ){ ++ sqlite3_free(pCsr->aPage); ++ pCsr->aPage = 0; ++ if( pCsr->bOnePage ) return SQLITE_OK; ++ pCsr->iPgno++; ++ }else{ ++ return SQLITE_OK; ++ } ++ }else{ ++ /* If there is no record loaded, load it now. */ ++ if( pCsr->pRec==0 ){ ++ int bHasRowid = 0; ++ int nPointer = 0; ++ sqlite3_int64 nPayload = 0; ++ sqlite3_int64 nHdr = 0; ++ int iHdr; ++ int U, X; ++ int nLocal; ++ ++ switch( pCsr->aPage[iOff] ){ ++ case 0x02: ++ nPointer = 4; ++ break; ++ case 0x0a: ++ break; ++ case 0x0d: ++ bHasRowid = 1; ++ break; ++ default: ++ /* This is not a b-tree page with records on it. Continue. */ ++ pCsr->iCell = pCsr->nCell; ++ break; ++ } ++ ++ if( pCsr->iCell>=pCsr->nCell ){ ++ bNextPage = 1; ++ }else{ ++ ++ iOff += 8 + nPointer + pCsr->iCell*2; ++ if( iOff>pCsr->nPage ){ ++ bNextPage = 1; ++ }else{ ++ iOff = get_uint16(&pCsr->aPage[iOff]); ++ } ++ ++ /* For an interior node cell, skip past the child-page number */ ++ iOff += nPointer; ++ ++ /* Load the "byte of payload including overflow" field */ ++ if( bNextPage || iOff>pCsr->nPage ){ ++ bNextPage = 1; ++ }else{ ++ iOff += dbdataGetVarint(&pCsr->aPage[iOff], &nPayload); ++ } ++ ++ /* If this is a leaf intkey cell, load the rowid */ ++ if( bHasRowid && !bNextPage && iOff<pCsr->nPage ){ ++ iOff += dbdataGetVarint(&pCsr->aPage[iOff], &pCsr->iIntkey); ++ } ++ ++ /* Figure out how much data to read from the local page */ ++ U = pCsr->nPage; ++ if( bHasRowid ){ ++ X = U-35; ++ }else{ ++ X = ((U-12)*64/255)-23; ++ } ++ if( nPayload<=X ){ ++ nLocal = nPayload; ++ }else{ ++ int M, K; ++ M = ((U-12)*32/255)-23; ++ K = M+((nPayload-M)%(U-4)); ++ if( K<=X ){ ++ nLocal = K; ++ }else{ ++ nLocal = M; ++ } ++ } ++ ++ if( bNextPage || nLocal+iOff>pCsr->nPage ){ ++ bNextPage = 1; ++ }else{ ++ ++ /* Allocate space for payload. And a bit more to catch small buffer ++ ** overruns caused by attempting to read a varint or similar from ++ ** near the end of a corrupt record. */ ++ pCsr->pRec = (u8*)sqlite3_malloc64(nPayload+DBDATA_PADDING_BYTES); ++ if( pCsr->pRec==0 ) return SQLITE_NOMEM; ++ memset(pCsr->pRec, 0, nPayload+DBDATA_PADDING_BYTES); ++ pCsr->nRec = nPayload; ++ ++ /* Load the nLocal bytes of payload */ ++ memcpy(pCsr->pRec, &pCsr->aPage[iOff], nLocal); ++ iOff += nLocal; ++ ++ /* Load content from overflow pages */ ++ if( nPayload>nLocal ){ ++ sqlite3_int64 nRem = nPayload - nLocal; ++ unsigned int pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); ++ while( nRem>0 ){ ++ u8 *aOvfl = 0; ++ int nOvfl = 0; ++ int nCopy; ++ rc = dbdataLoadPage(pCsr, pgnoOvfl, &aOvfl, &nOvfl); ++ assert( rc!=SQLITE_OK || aOvfl==0 || nOvfl==pCsr->nPage ); ++ if( rc!=SQLITE_OK ) return rc; ++ if( aOvfl==0 ) break; ++ ++ nCopy = U-4; ++ if( nCopy>nRem ) nCopy = nRem; ++ memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy); ++ nRem -= nCopy; ++ ++ pgnoOvfl = get_uint32(aOvfl); ++ sqlite3_free(aOvfl); ++ } ++ } ++ ++ iHdr = dbdataGetVarint(pCsr->pRec, &nHdr); ++ pCsr->nHdr = nHdr; ++ pCsr->pHdrPtr = &pCsr->pRec[iHdr]; ++ pCsr->pPtr = &pCsr->pRec[pCsr->nHdr]; ++ pCsr->iField = (bHasRowid ? -1 : 0); ++ } ++ } ++ }else{ ++ pCsr->iField++; ++ if( pCsr->iField>0 ){ ++ sqlite3_int64 iType; ++ if( pCsr->pHdrPtr>&pCsr->pRec[pCsr->nRec] ){ ++ bNextPage = 1; ++ }else{ ++ pCsr->pHdrPtr += dbdataGetVarint(pCsr->pHdrPtr, &iType); ++ pCsr->pPtr += dbdataValueBytes(iType); ++ } ++ } ++ } ++ ++ if( bNextPage ){ ++ sqlite3_free(pCsr->aPage); ++ sqlite3_free(pCsr->pRec); ++ pCsr->aPage = 0; ++ pCsr->pRec = 0; ++ if( pCsr->bOnePage ) return SQLITE_OK; ++ pCsr->iPgno++; ++ }else{ ++ if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->pRec[pCsr->nHdr] ){ ++ return SQLITE_OK; ++ } ++ ++ /* Advance to the next cell. The next iteration of the loop will load ++ ** the record and so on. */ ++ sqlite3_free(pCsr->pRec); ++ pCsr->pRec = 0; ++ pCsr->iCell++; ++ } ++ } ++ } ++ ++ assert( !"can't get here" ); ++ return SQLITE_OK; ++} ++ ++/* ++** Return true if the cursor is at EOF. ++*/ ++static int dbdataEof(sqlite3_vtab_cursor *pCursor){ ++ DbdataCursor *pCsr = (DbdataCursor*)pCursor; ++ return pCsr->aPage==0; ++} ++ ++/* ++** Determine the size in pages of database zSchema (where zSchema is ++** "main", "temp" or the name of an attached database) and set ++** pCsr->szDb accordingly. If successful, return SQLITE_OK. Otherwise, ++** an SQLite error code. ++*/ ++static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ ++ DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab; ++ char *zSql = 0; ++ int rc, rc2; ++ sqlite3_stmt *pStmt = 0; ++ ++ zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema); ++ if( zSql==0 ) return SQLITE_NOMEM; ++ rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0); ++ sqlite3_free(zSql); ++ if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ ++ pCsr->szDb = sqlite3_column_int(pStmt, 0); ++ } ++ rc2 = sqlite3_finalize(pStmt); ++ if( rc==SQLITE_OK ) rc = rc2; ++ return rc; ++} ++ ++/* ++** xFilter method for sqlite_dbdata and sqlite_dbptr. ++*/ ++static int dbdataFilter( ++ sqlite3_vtab_cursor *pCursor, ++ int idxNum, const char *idxStr, ++ int argc, sqlite3_value **argv ++){ ++ DbdataCursor *pCsr = (DbdataCursor*)pCursor; ++ DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; ++ int rc = SQLITE_OK; ++ const char *zSchema = "main"; ++ ++ dbdataResetCursor(pCsr); ++ assert( pCsr->iPgno==1 ); ++ if( idxNum & 0x01 ){ ++ zSchema = (const char*)sqlite3_value_text(argv[0]); ++ } ++ if( idxNum & 0x02 ){ ++ pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); ++ pCsr->bOnePage = 1; ++ }else{ ++ pCsr->nPage = dbdataDbsize(pCsr, zSchema); ++ rc = dbdataDbsize(pCsr, zSchema); ++ } ++ ++ if( rc==SQLITE_OK ){ ++ if( pTab->pStmt ){ ++ pCsr->pStmt = pTab->pStmt; ++ pTab->pStmt = 0; ++ }else{ ++ rc = sqlite3_prepare_v2(pTab->db, ++ "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, ++ &pCsr->pStmt, 0 ++ ); ++ } ++ } ++ if( rc==SQLITE_OK ){ ++ rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT); ++ }else{ ++ pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); ++ } ++ if( rc==SQLITE_OK ){ ++ rc = dbdataNext(pCursor); ++ } ++ return rc; ++} ++ ++/* ++** Return a column for the sqlite_dbdata or sqlite_dbptr table. ++*/ ++static int dbdataColumn( ++ sqlite3_vtab_cursor *pCursor, ++ sqlite3_context *ctx, ++ int i ++){ ++ DbdataCursor *pCsr = (DbdataCursor*)pCursor; ++ DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; ++ if( pTab->bPtr ){ ++ switch( i ){ ++ case DBPTR_COLUMN_PGNO: ++ sqlite3_result_int64(ctx, pCsr->iPgno); ++ break; ++ case DBPTR_COLUMN_CHILD: { ++ int iOff = pCsr->iPgno==1 ? 100 : 0; ++ if( pCsr->iCell<0 ){ ++ iOff += 8; ++ }else{ ++ iOff += 12 + pCsr->iCell*2; ++ if( iOff>pCsr->nPage ) return SQLITE_OK; ++ iOff = get_uint16(&pCsr->aPage[iOff]); ++ } ++ if( iOff<=pCsr->nPage ){ ++ sqlite3_result_int64(ctx, get_uint32(&pCsr->aPage[iOff])); ++ } ++ break; ++ } ++ } ++ }else{ ++ switch( i ){ ++ case DBDATA_COLUMN_PGNO: ++ sqlite3_result_int64(ctx, pCsr->iPgno); ++ break; ++ case DBDATA_COLUMN_CELL: ++ sqlite3_result_int(ctx, pCsr->iCell); ++ break; ++ case DBDATA_COLUMN_FIELD: ++ sqlite3_result_int(ctx, pCsr->iField); ++ break; ++ case DBDATA_COLUMN_VALUE: { ++ if( pCsr->iField<0 ){ ++ sqlite3_result_int64(ctx, pCsr->iIntkey); ++ }else{ ++ sqlite3_int64 iType; ++ dbdataGetVarint(pCsr->pHdrPtr, &iType); ++ dbdataValue( ++ ctx, iType, pCsr->pPtr, &pCsr->pRec[pCsr->nRec] - pCsr->pPtr ++ ); ++ } ++ break; ++ } ++ } ++ } ++ return SQLITE_OK; ++} ++ ++/* ++** Return the rowid for an sqlite_dbdata or sqlite_dptr table. ++*/ ++static int dbdataRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ ++ DbdataCursor *pCsr = (DbdataCursor*)pCursor; ++ *pRowid = pCsr->iRowid; ++ return SQLITE_OK; ++} ++ ++ ++/* ++** Invoke this routine to register the "sqlite_dbdata" virtual table module ++*/ ++static int sqlite3DbdataRegister(sqlite3 *db){ ++ static sqlite3_module dbdata_module = { ++ 0, /* iVersion */ ++ 0, /* xCreate */ ++ dbdataConnect, /* xConnect */ ++ dbdataBestIndex, /* xBestIndex */ ++ dbdataDisconnect, /* xDisconnect */ ++ 0, /* xDestroy */ ++ dbdataOpen, /* xOpen - open a cursor */ ++ dbdataClose, /* xClose - close a cursor */ ++ dbdataFilter, /* xFilter - configure scan constraints */ ++ dbdataNext, /* xNext - advance a cursor */ ++ dbdataEof, /* xEof - check for end of scan */ ++ dbdataColumn, /* xColumn - read data */ ++ dbdataRowid, /* xRowid - read data */ ++ 0, /* xUpdate */ ++ 0, /* xBegin */ ++ 0, /* xSync */ ++ 0, /* xCommit */ ++ 0, /* xRollback */ ++ 0, /* xFindMethod */ ++ 0, /* xRename */ ++ 0, /* xSavepoint */ ++ 0, /* xRelease */ ++ 0, /* xRollbackTo */ ++ 0 /* xShadowName */ ++ }; ++ ++ int rc = sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0); ++ if( rc==SQLITE_OK ){ ++ rc = sqlite3_create_module(db, "sqlite_dbptr", &dbdata_module, (void*)1); ++ } ++ return rc; ++} ++ ++#ifdef _WIN32 ++ ++#endif ++int sqlite3_dbdata_init( ++ sqlite3 *db, ++ char **pzErrMsg, ++ const sqlite3_api_routines *pApi ++){ ++ SQLITE_EXTENSION_INIT2(pApi); ++ return sqlite3DbdataRegister(db); ++} ++ ++/************************* End ../ext/misc/dbdata.c ********************/ ++#endif ++ ++#if defined(SQLITE_ENABLE_SESSION) ++/* ++** State information for a single open session ++*/ ++typedef struct OpenSession OpenSession; ++struct OpenSession { ++ char *zName; /* Symbolic name for this session */ ++ int nFilter; /* Number of xFilter rejection GLOB patterns */ ++ char **azFilter; /* Array of xFilter rejection GLOB patterns */ ++ sqlite3_session *p; /* The open session */ ++}; ++#endif ++ ++/* ++** Shell output mode information from before ".explain on", ++** saved so that it can be restored by ".explain off" ++*/ ++typedef struct SavedModeInfo SavedModeInfo; ++struct SavedModeInfo { ++ int valid; /* Is there legit data in here? */ ++ int mode; /* Mode prior to ".explain on" */ ++ int showHeader; /* The ".header" setting prior to ".explain on" */ ++ int colWidth[100]; /* Column widths prior to ".explain on" */ ++}; ++ ++typedef struct ExpertInfo ExpertInfo; ++struct ExpertInfo { ++ sqlite3expert *pExpert; ++ int bVerbose; ++}; ++ ++/* A single line in the EQP output */ ++typedef struct EQPGraphRow EQPGraphRow; ++struct EQPGraphRow { ++ int iEqpId; /* ID for this row */ ++ int iParentId; /* ID of the parent row */ ++ EQPGraphRow *pNext; /* Next row in sequence */ ++ char zText[1]; /* Text to display for this row */ ++}; ++ ++/* All EQP output is collected into an instance of the following */ ++typedef struct EQPGraph EQPGraph; ++struct EQPGraph { ++ EQPGraphRow *pRow; /* Linked list of all rows of the EQP output */ ++ EQPGraphRow *pLast; /* Last element of the pRow list */ ++ char zPrefix[100]; /* Graph prefix */ ++}; ++ ++/* ++** State information about the database connection is contained in an ++** instance of the following structure. ++*/ ++typedef struct ShellState ShellState; ++struct ShellState { ++ sqlite3 *db; /* The database */ ++ u8 autoExplain; /* Automatically turn on .explain mode */ ++ u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */ ++ u8 autoEQPtest; /* autoEQP is in test mode */ ++ u8 autoEQPtrace; /* autoEQP is in trace mode */ ++ u8 statsOn; /* True to display memory stats before each finalize */ ++ u8 scanstatsOn; /* True to display scan stats before each finalize */ ++ u8 openMode; /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */ ++ u8 doXdgOpen; /* Invoke start/open/xdg-open in output_reset() */ ++ u8 nEqpLevel; /* Depth of the EQP output graph */ ++ u8 eTraceType; /* SHELL_TRACE_* value for type of trace */ ++ unsigned mEqpLines; /* Mask of veritical lines in the EQP output graph */ ++ int outCount; /* Revert to stdout when reaching zero */ ++ int cnt; /* Number of records displayed so far */ ++ int lineno; /* Line number of last line read from in */ ++ int openFlags; /* Additional flags to open. (SQLITE_OPEN_NOFOLLOW) */ ++ FILE *in; /* Read commands from this stream */ ++ FILE *out; /* Write results here */ ++ FILE *traceOut; /* Output for sqlite3_trace() */ ++ int nErr; /* Number of errors seen */ ++ int mode; /* An output mode setting */ ++ int modePrior; /* Saved mode */ ++ int cMode; /* temporary output mode for the current query */ ++ int normalMode; /* Output mode before ".explain on" */ ++ int writableSchema; /* True if PRAGMA writable_schema=ON */ ++ int showHeader; /* True to show column names in List or Column mode */ ++ int nCheck; /* Number of ".check" commands run */ ++ unsigned nProgress; /* Number of progress callbacks encountered */ ++ unsigned mxProgress; /* Maximum progress callbacks before failing */ ++ unsigned flgProgress; /* Flags for the progress callback */ ++ unsigned shellFlgs; /* Various flags */ ++ unsigned priorShFlgs; /* Saved copy of flags */ ++ sqlite3_int64 szMax; /* --maxsize argument to .open */ ++ char *zDestTable; /* Name of destination table when MODE_Insert */ ++ char *zTempFile; /* Temporary file that might need deleting */ ++ char zTestcase[30]; /* Name of current test case */ ++ char colSeparator[20]; /* Column separator character for several modes */ ++ char rowSeparator[20]; /* Row separator character for MODE_Ascii */ ++ char colSepPrior[20]; /* Saved column separator */ ++ char rowSepPrior[20]; /* Saved row separator */ ++ int colWidth[100]; /* Requested width of each column when in column mode*/ ++ int actualWidth[100]; /* Actual width of each column */ ++ char nullValue[20]; /* The text to print when a NULL comes back from ++ ** the database */ ++ char outfile[FILENAME_MAX]; /* Filename for *out */ ++ const char *zDbFilename; /* name of the database file */ ++ char *zFreeOnClose; /* Filename to free when closing */ ++ const char *zVfs; /* Name of VFS to use */ ++ sqlite3_stmt *pStmt; /* Current statement if any. */ ++ FILE *pLog; /* Write log output here */ ++ int *aiIndent; /* Array of indents used in MODE_Explain */ ++ int nIndent; /* Size of array aiIndent[] */ ++ int iIndent; /* Index of current op in aiIndent[] */ ++ EQPGraph sGraph; /* Information for the graphical EXPLAIN QUERY PLAN */ ++#if defined(SQLITE_ENABLE_SESSION) ++ int nSession; /* Number of active sessions */ ++ OpenSession aSession[4]; /* Array of sessions. [0] is in focus. */ ++#endif ++ ExpertInfo expert; /* Valid if previous command was ".expert OPT..." */ ++}; ++ ++ ++/* Allowed values for ShellState.autoEQP ++*/ ++#define AUTOEQP_off 0 /* Automatic EXPLAIN QUERY PLAN is off */ ++#define AUTOEQP_on 1 /* Automatic EQP is on */ ++#define AUTOEQP_trigger 2 /* On and also show plans for triggers */ + #define AUTOEQP_full 3 /* Show full EXPLAIN */ + + /* Allowed values for ShellState.openMode +@@ -8935,12 +9995,12 @@ + } + sz = sqlite3_value_bytes(argv[0]); + if( bBin ){ +- x = fwrite(sqlite3_value_blob(argv[0]), 1, sz, f); ++ x = fwrite(sqlite3_value_blob(argv[0]), 1, (size_t)sz, f); + }else{ + const char *z = (const char*)sqlite3_value_text(argv[0]); + /* Remember whether or not the value originally contained \r\n */ + if( z && strstr(z,"\r\n")!=0 ) hasCRNL = 1; +- x = fwrite(sqlite3_value_text(argv[0]), 1, sz, f); ++ x = fwrite(sqlite3_value_text(argv[0]), 1, (size_t)sz, f); + } + fclose(f); + f = 0; +@@ -8968,12 +10028,12 @@ + fseek(f, 0, SEEK_END); + sz = ftell(f); + rewind(f); +- p = sqlite3_malloc64( sz+(bBin==0) ); ++ p = sqlite3_malloc64( sz+1 ); + if( p==0 ){ + sqlite3_result_error_nomem(context); + goto edit_func_end; + } +- x = fread(p, 1, sz, f); ++ x = fread(p, 1, (size_t)sz, f); + fclose(f); + f = 0; + if( x!=sz ){ +@@ -9015,11 +10075,13 @@ + */ + static void outputModePush(ShellState *p){ + p->modePrior = p->mode; ++ p->priorShFlgs = p->shellFlgs; + memcpy(p->colSepPrior, p->colSeparator, sizeof(p->colSeparator)); + memcpy(p->rowSepPrior, p->rowSeparator, sizeof(p->rowSeparator)); + } + static void outputModePop(ShellState *p){ + p->mode = p->modePrior; ++ p->shellFlgs = p->priorShFlgs; + memcpy(p->colSeparator, p->colSepPrior, sizeof(p->colSeparator)); + memcpy(p->rowSeparator, p->rowSepPrior, sizeof(p->rowSeparator)); + } +@@ -9360,6 +10422,8 @@ + ** in FTS3/4/5 into CREATE TABLE IF NOT EXISTS statements. + */ + static void printSchemaLine(FILE *out, const char *z, const char *zTail){ ++ if( z==0 ) return; ++ if( zTail==0 ) return; + if( sqlite3_strglob("CREATE TABLE ['\"]*", z)==0 ){ + utf8_printf(out, "CREATE TABLE IF NOT EXISTS %s%s", z+13, zTail); + }else{ +@@ -9443,7 +10507,8 @@ + for(pRow = eqp_next_row(p, iEqpId, 0); pRow; pRow = pNext){ + pNext = eqp_next_row(p, iEqpId, pRow); + z = pRow->zText; +- utf8_printf(p->out, "%s%s%s\n", p->sGraph.zPrefix, pNext ? "|--" : "`--", z); ++ utf8_printf(p->out, "%s%s%s\n", p->sGraph.zPrefix, ++ pNext ? "|--" : "`--", z); + if( n<(int)sizeof(p->sGraph.zPrefix)-7 ){ + memcpy(&p->sGraph.zPrefix[n], pNext ? "| " : " ", 4); + eqp_render_level(p, pRow->iEqpId); +@@ -9531,19 +10596,22 @@ + const int *colWidth; + int showHdr; + char *rowSep; ++ int nWidth; + if( p->cMode==MODE_Column ){ + colWidth = p->colWidth; ++ nWidth = ArraySize(p->colWidth); + showHdr = p->showHeader; + rowSep = p->rowSeparator; + }else{ + colWidth = aExplainWidths; ++ nWidth = ArraySize(aExplainWidths); + showHdr = 1; + rowSep = SEP_Row; + } + if( p->cnt++==0 ){ + for(i=0; i<nArg; i++){ + int w, n; +- if( i<ArraySize(p->colWidth) ){ ++ if( i<nWidth ){ + w = colWidth[i]; + }else{ + w = 0; +@@ -9634,7 +10702,7 @@ + while( j>0 && IsSpace(z[j-1]) ){ j--; } + z[j] = 0; + if( strlen30(z)>=79 ){ +- for(i=j=0; (c = z[i])!=0; i++){ /* Copy changes from z[i] back to z[j] */ ++ for(i=j=0; (c = z[i])!=0; i++){ /* Copy from z[i] back to z[j] */ + if( c==cEnd ){ + cEnd = 0; + }else if( c=='"' || c=='\'' || c=='`' ){ +@@ -9978,8 +11046,7 @@ + */ + static int run_table_dump_query( + ShellState *p, /* Query context */ +- const char *zSelect, /* SELECT statement to extract content */ +- const char *zFirstRow /* Print before first row, if not NULL */ ++ const char *zSelect /* SELECT statement to extract content */ + ){ + sqlite3_stmt *pSelect; + int rc; +@@ -9996,10 +11063,6 @@ + rc = sqlite3_step(pSelect); + nResult = sqlite3_column_count(pSelect); + while( rc==SQLITE_ROW ){ +- if( zFirstRow ){ +- utf8_printf(p->out, "%s", zFirstRow); +- zFirstRow = 0; +- } + z = (const char*)sqlite3_column_text(pSelect, 0); + utf8_printf(p->out, "%s", z); + for(i=1; i<nResult; i++){ +@@ -10213,7 +11276,7 @@ + raw_printf(pArg->out, "Autoindex Inserts: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); + raw_printf(pArg->out, "Virtual Machine Steps: %d\n", iCur); +- iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_REPREPARE, bReset); ++ iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_REPREPARE,bReset); + raw_printf(pArg->out, "Reprepare operations: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_RUN, bReset); + raw_printf(pArg->out, "Number of times run: %d\n", iCur); +@@ -10433,6 +11496,9 @@ + /* Create the TEMP table used to store parameter bindings */ + static void bind_table_init(ShellState *p){ + int wrSchema = 0; ++ int defensiveMode = 0; ++ sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, -1, &defensiveMode); ++ sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, -1, &wrSchema); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, 1, 0); + sqlite3_exec(p->db, +@@ -10442,6 +11508,7 @@ + ") WITHOUT ROWID;", + 0, 0, 0); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, wrSchema, 0); ++ sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, defensiveMode, 0); + } + + /* +@@ -10452,9 +11519,9 @@ + ** CREATE TEMP TABLE sqlite_parameters(key TEXT PRIMARY KEY, value) + ** WITHOUT ROWID; + ** +-** No bindings occur if this table does not exist. The special character '$' +-** is included in the table name to help prevent collisions with actual tables. +-** The table must be in the TEMP schema. ++** No bindings occur if this table does not exist. The name of the table ++** begins with "sqlite_" so that it will not collide with ordinary application ++** tables. The table must be in the TEMP schema. + */ + static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ + int nVar; +@@ -10758,6 +11825,7 @@ + const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3); + int iEqpId = sqlite3_column_int(pExplain, 0); + int iParentId = sqlite3_column_int(pExplain, 1); ++ if( zEQPLine==0 ) zEQPLine = ""; + if( zEQPLine[0]=='-' ) eqp_render(pArg); + eqp_append(pArg, iEqpId, iParentId, zEQPLine); + } +@@ -11136,20 +12204,20 @@ + ".archive ... Manage SQL archives", + " Each command must have exactly one of the following options:", + " -c, --create Create a new archive", +- " -u, --update Add files or update files with changed mtime", +- " -i, --insert Like -u but always add even if mtime unchanged", ++ " -u, --update Add or update files with changed mtime", ++ " -i, --insert Like -u but always add even if unchanged", + " -t, --list List contents of archive", + " -x, --extract Extract files from archive", + " Optional arguments:", + " -v, --verbose Print each filename as it is processed", +- " -f FILE, --file FILE Operate on archive FILE (default is current db)", +- " -a FILE, --append FILE Operate on FILE opened using the apndvfs VFS", +- " -C DIR, --directory DIR Change to directory DIR to read/extract files", ++ " -f FILE, --file FILE Use archive FILE (default is current db)", ++ " -a FILE, --append FILE Open FILE using the apndvfs VFS", ++ " -C DIR, --directory DIR Read/extract files from directory DIR", + " -n, --dryrun Show the SQL that would have occurred", + " Examples:", +- " .ar -cf archive.sar foo bar # Create archive.sar from files foo and bar", +- " .ar -tf archive.sar # List members of archive.sar", +- " .ar -xvf archive.sar # Verbosely extract files from archive.sar", ++ " .ar -cf ARCHIVE foo bar # Create ARCHIVE from files foo and bar", ++ " .ar -tf ARCHIVE # List members of ARCHIVE", ++ " .ar -xvf ARCHIVE # Verbosely extract files from ARCHIVE", + " See also:", + " http://sqlite.org/cli.html#sqlar_archive_support", + #endif +@@ -11158,7 +12226,7 @@ + #endif + ".backup ?DB? FILE Backup DB (default \"main\") to FILE", + " --append Use the appendvfs", +- " --async Write to FILE without a journal and without fsync()", ++ " --async Write to FILE without journal and fsync()", + ".bail on|off Stop after hitting an error. Default OFF", + ".binary on|off Turn binary output on or off. Default OFF", + ".cd DIRECTORY Change the working directory to DIRECTORY", +@@ -11168,29 +12236,44 @@ + ".databases List names and files of attached databases", + ".dbconfig ?op? ?val? List or change sqlite3_db_config() options", + ".dbinfo ?DB? Show status information about the database", +- ".dump ?TABLE? ... Render all database content as SQL", ++ ".dump ?TABLE? Render database content as SQL", + " Options:", + " --preserve-rowids Include ROWID values in the output", + " --newlines Allow unescaped newline characters in output", +- " TABLE is LIKE pattern for the tables to dump", ++ " TABLE is a LIKE pattern for the tables to dump", ++ " Additional LIKE patterns can be given in subsequent arguments", + ".echo on|off Turn command echo on or off", + ".eqp on|off|full|... Enable or disable automatic EXPLAIN QUERY PLAN", + " Other Modes:", + #ifdef SQLITE_DEBUG + " test Show raw EXPLAIN QUERY PLAN output", +- " trace Like \"full\" but also enable \"PRAGMA vdbe_trace\"", ++ " trace Like \"full\" but enable \"PRAGMA vdbe_trace\"", + #endif + " trigger Like \"full\" but also show trigger bytecode", +- ".excel Display the output of next command in a spreadsheet", ++ ".excel Display the output of next command in spreadsheet", ++ " --bom Put a UTF8 byte-order mark on intermediate file", + ".exit ?CODE? Exit this program with return-code CODE", +- ".expert EXPERIMENTAL. Suggest indexes for specified queries", +-/* Because explain mode comes on automatically now, the ".explain" mode +-** is removed from the help screen. It is still supported for legacy, however */ +-/*".explain ?on|off|auto? Turn EXPLAIN output mode on or off or to automatic",*/ ++ ".expert EXPERIMENTAL. Suggest indexes for queries", ++ ".explain ?on|off|auto? Change the EXPLAIN formatting mode. Default: auto", ++ ".filectrl CMD ... Run various sqlite3_file_control() operations", ++ " --schema SCHEMA Use SCHEMA instead of \"main\"", ++ " --help Show CMD details", + ".fullschema ?--indent? Show schema and the content of sqlite_stat tables", + ".headers on|off Turn display of headers on or off", + ".help ?-all? ?PATTERN? Show help text for PATTERN", + ".import FILE TABLE Import data from FILE into TABLE", ++ " Options:", ++ " --ascii Use \\037 and \\036 as column and row separators", ++ " --csv Use , and \\n as column and row separators", ++ " --skip N Skip the first N rows of input", ++ " -v \"Verbose\" - increase auxiliary output", ++ " Notes:", ++ " * If TABLE does not exist, it is created. The first row of input", ++ " determines the column names.", ++ " * If neither --csv or --ascii are used, the input mode is derived", ++ " from the \".mode\" output mode", ++ " * If FILE begins with \"|\" then it is a command that generates the", ++ " input text.", + #ifndef SQLITE_OMIT_TEST_CONTROL + ".imposter INDEX TABLE Create imposter table TABLE on index INDEX", + #endif +@@ -11221,30 +12304,38 @@ + " tabs Tab-separated values", + " tcl TCL list elements", + ".nullvalue STRING Use STRING in place of NULL values", +- ".once (-e|-x|FILE) Output for the next SQL command only to FILE", ++ ".once ?OPTIONS? ?FILE? Output for the next SQL command only to FILE", + " If FILE begins with '|' then open as a pipe", +- " Other options:", +- " -e Invoke system text editor", +- " -x Open in a spreadsheet", ++ " --bom Put a UTF8 byte-order mark at the beginning", ++ " -e Send output to the system text editor", ++ " -x Send output as CSV to a spreadsheet (same as \".excel\")", ++#ifdef SQLITE_DEBUG ++ ".oom [--repeat M] [N] Simulate an OOM error on the N-th allocation", ++#endif + ".open ?OPTIONS? ?FILE? Close existing database and reopen FILE", + " Options:", + " --append Use appendvfs to append database to the end of FILE", + #ifdef SQLITE_ENABLE_DESERIALIZE + " --deserialize Load into memory useing sqlite3_deserialize()", +- " --hexdb Load the output of \"dbtotxt\" as an in-memory database", ++ " --hexdb Load the output of \"dbtotxt\" as an in-memory db", + " --maxsize N Maximum size for --hexdb or --deserialized database", + #endif + " --new Initialize FILE to an empty database", ++ " --nofollow Do not follow symbolic links", + " --readonly Open FILE readonly", + " --zip FILE is a ZIP archive", + ".output ?FILE? Send output to FILE or stdout if FILE is omitted", +- " If FILE begins with '|' then open it as a pipe.", ++ " If FILE begins with '|' then open it as a pipe.", ++ " Options:", ++ " --bom Prefix output with a UTF8 byte-order mark", ++ " -e Send output to the system text editor", ++ " -x Send output as CSV to a spreadsheet", + ".parameter CMD ... Manage SQL parameter bindings", + " clear Erase all bindings", + " init Initialize the TEMP table that holds bindings", + " list List the current parameter bindings", + " set PARAMETER VALUE Given SQL parameter PARAMETER a value of VALUE", +- " PARAMETER should start with '$', ':', '@', or '?'", ++ " PARAMETER should start with one of: $ : @ ?", + " unset PARAMETER Remove PARAMETER from the binding table", + ".print STRING... Print literal STRING", + #ifndef SQLITE_OMIT_PROGRESS_CALLBACK +@@ -11257,6 +12348,14 @@ + ".prompt MAIN CONTINUE Replace the standard prompts", + ".quit Exit this program", + ".read FILE Read input from FILE", ++#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) ++ ".recover Recover as much data as possible from corrupt db.", ++ " --freelist-corrupt Assume the freelist is corrupt", ++ " --recovery-db NAME Store recovery metadata in database file NAME", ++ " --lost-and-found TABLE Alternative name for the lost-and-found table", ++ " --no-rowids Do not attempt to recover rowid values", ++ " that are not also INTEGER PRIMARY KEYs", ++#endif + ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE", + ".save FILE Write in-memory database into FILE", + ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off", +@@ -11287,7 +12386,7 @@ + " Options:", + " --schema Also hash the sqlite_master table", + " --sha3-224 Use the sha3-224 algorithm", +- " --sha3-256 Use the sha3-256 algorithm. This is the default.", ++ " --sha3-256 Use the sha3-256 algorithm (default)", + " --sha3-384 Use the sha3-384 algorithm", + " --sha3-512 Use the sha3-512 algorithm", + " Any other argument is a LIKE pattern for tables to hash", +@@ -11301,6 +12400,8 @@ + #endif + ".tables ?TABLE? List names of tables matching LIKE pattern TABLE", + ".testcase NAME Begin redirecting output to 'testcase-out.txt'", ++ ".testctrl CMD ... Run various sqlite3_test_control() operations", ++ " Run \".testctrl\" with no arguments for details", + ".timeout MS Try opening locked tables for MS milliseconds", + ".timer on|off Turn SQL timer on or off", + #ifndef SQLITE_OMIT_TRACE +@@ -11319,6 +12420,10 @@ + " --row Trace each row (SQLITE_TRACE_ROW)", + " --close Trace connection close (SQLITE_TRACE_CLOSE)", + #endif /* SQLITE_OMIT_TRACE */ ++#ifdef SQLITE_DEBUG ++ ".unmodule NAME ... Unregister virtual table modules", ++ " --allexcept Unregister everything except those named", ++#endif + ".vfsinfo ?AUX? Information about the top-level VFS", + ".vfslist List all available VFSes", + ".vfsname ?AUX? Print the name of the VFS stack", +@@ -11344,6 +12449,7 @@ + || zPattern[0]=='0' + || strcmp(zPattern,"-a")==0 + || strcmp(zPattern,"-all")==0 ++ || strcmp(zPattern,"--all")==0 + ){ + /* Show all commands, but only one line per command */ + if( zPattern==0 ) zPattern = ""; +@@ -11541,7 +12647,7 @@ + int j, k; + int rc; + FILE *in; +- unsigned char x[16]; ++ unsigned int x[16]; + char zLine[1000]; + if( p->zDbFilename ){ + in = fopen(p->zDbFilename, "r"); +@@ -11553,14 +12659,17 @@ + }else{ + in = p->in; + nLine = p->lineno; ++ if( in==0 ) in = stdin; + } + *pnData = 0; + nLine++; + if( fgets(zLine, sizeof(zLine), in)==0 ) goto readHexDb_error; + rc = sscanf(zLine, "| size %d pagesize %d", &n, &pgsz); + if( rc!=2 ) goto readHexDb_error; +- if( n<=0 ) goto readHexDb_error; +- a = sqlite3_malloc( n ); ++ if( n<0 ) goto readHexDb_error; ++ if( pgsz<512 || pgsz>65536 || (pgsz&(pgsz-1))!=0 ) goto readHexDb_error; ++ n = (n+pgsz-1)&~(pgsz-1); /* Round n up to the next multiple of pgsz */ ++ a = sqlite3_malloc( n ? n : 1 ); + if( a==0 ){ + utf8_printf(stderr, "Out of memory!\n"); + goto readHexDb_error; +@@ -11579,40 +12688,178 @@ + if( strncmp(zLine, "| end ", 6)==0 ){ + break; + } +- rc = sscanf(zLine,"| %d: %hhx %hhx %hhx %hhx %hhx %hhx %hhx %hhx" +- " %hhx %hhx %hhx %hhx %hhx %hhx %hhx %hhx", ++ rc = sscanf(zLine,"| %d: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x", + &j, &x[0], &x[1], &x[2], &x[3], &x[4], &x[5], &x[6], &x[7], + &x[8], &x[9], &x[10], &x[11], &x[12], &x[13], &x[14], &x[15]); + if( rc==17 ){ + k = iOffset+j; + if( k+16<=n ){ +- memcpy(a+k, x, 16); ++ int ii; ++ for(ii=0; ii<16; ii++) a[k+ii] = x[ii]&0xff; ++ } ++ } ++ } ++ *pnData = n; ++ if( in!=p->in ){ ++ fclose(in); ++ }else{ ++ p->lineno = nLine; ++ } ++ return a; ++ ++readHexDb_error: ++ if( in!=p->in ){ ++ fclose(in); ++ }else{ ++ while( fgets(zLine, sizeof(zLine), p->in)!=0 ){ ++ nLine++; ++ if(strncmp(zLine, "| end ", 6)==0 ) break; ++ } ++ p->lineno = nLine; ++ } ++ sqlite3_free(a); ++ utf8_printf(stderr,"Error on line %d of --hexdb input\n", nLine); ++ return 0; ++} ++#endif /* SQLITE_ENABLE_DESERIALIZE */ ++ ++/* ++** Scalar function "shell_int32". The first argument to this function ++** must be a blob. The second a non-negative integer. This function ++** reads and returns a 32-bit big-endian integer from byte ++** offset (4*<arg2>) of the blob. ++*/ ++static void shellInt32( ++ sqlite3_context *context, ++ int argc, ++ sqlite3_value **argv ++){ ++ const unsigned char *pBlob; ++ int nBlob; ++ int iInt; ++ ++ UNUSED_PARAMETER(argc); ++ nBlob = sqlite3_value_bytes(argv[0]); ++ pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); ++ iInt = sqlite3_value_int(argv[1]); ++ ++ if( iInt>=0 && (iInt+1)*4<=nBlob ){ ++ const unsigned char *a = &pBlob[iInt*4]; ++ sqlite3_int64 iVal = ((sqlite3_int64)a[0]<<24) ++ + ((sqlite3_int64)a[1]<<16) ++ + ((sqlite3_int64)a[2]<< 8) ++ + ((sqlite3_int64)a[3]<< 0); ++ sqlite3_result_int64(context, iVal); ++ } ++} ++ ++/* ++** Scalar function "shell_idquote(X)" returns string X quoted as an identifier, ++** using "..." with internal double-quote characters doubled. ++*/ ++static void shellIdQuote( ++ sqlite3_context *context, ++ int argc, ++ sqlite3_value **argv ++){ ++ const char *zName = (const char*)sqlite3_value_text(argv[0]); ++ UNUSED_PARAMETER(argc); ++ if( zName ){ ++ char *z = sqlite3_mprintf("\"%w\"", zName); ++ sqlite3_result_text(context, z, -1, sqlite3_free); ++ } ++} ++ ++/* ++** Scalar function "shell_escape_crnl" used by the .recover command. ++** The argument passed to this function is the output of built-in ++** function quote(). If the first character of the input is "'", ++** indicating that the value passed to quote() was a text value, ++** then this function searches the input for "\n" and "\r" characters ++** and adds a wrapper similar to the following: ++** ++** replace(replace(<input>, '\n', char(10), '\r', char(13)); ++** ++** Or, if the first character of the input is not "'", then a copy ++** of the input is returned. ++*/ ++static void shellEscapeCrnl( ++ sqlite3_context *context, ++ int argc, ++ sqlite3_value **argv ++){ ++ const char *zText = (const char*)sqlite3_value_text(argv[0]); ++ UNUSED_PARAMETER(argc); ++ if( zText[0]=='\'' ){ ++ int nText = sqlite3_value_bytes(argv[0]); ++ int i; ++ char zBuf1[20]; ++ char zBuf2[20]; ++ const char *zNL = 0; ++ const char *zCR = 0; ++ int nCR = 0; ++ int nNL = 0; ++ ++ for(i=0; zText[i]; i++){ ++ if( zNL==0 && zText[i]=='\n' ){ ++ zNL = unused_string(zText, "\\n", "\\012", zBuf1); ++ nNL = (int)strlen(zNL); ++ } ++ if( zCR==0 && zText[i]=='\r' ){ ++ zCR = unused_string(zText, "\\r", "\\015", zBuf2); ++ nCR = (int)strlen(zCR); ++ } ++ } ++ ++ if( zNL || zCR ){ ++ int iOut = 0; ++ i64 nMax = (nNL > nCR) ? nNL : nCR; ++ i64 nAlloc = nMax * nText + (nMax+64)*2; ++ char *zOut = (char*)sqlite3_malloc64(nAlloc); ++ if( zOut==0 ){ ++ sqlite3_result_error_nomem(context); ++ return; ++ } ++ ++ if( zNL && zCR ){ ++ memcpy(&zOut[iOut], "replace(replace(", 16); ++ iOut += 16; ++ }else{ ++ memcpy(&zOut[iOut], "replace(", 8); ++ iOut += 8; ++ } ++ for(i=0; zText[i]; i++){ ++ if( zText[i]=='\n' ){ ++ memcpy(&zOut[iOut], zNL, nNL); ++ iOut += nNL; ++ }else if( zText[i]=='\r' ){ ++ memcpy(&zOut[iOut], zCR, nCR); ++ iOut += nCR; ++ }else{ ++ zOut[iOut] = zText[i]; ++ iOut++; ++ } + } +- } +- } +- *pnData = n; +- if( in!=p->in ){ +- fclose(in); +- }else{ +- p->lineno = nLine; +- } +- return a; + +-readHexDb_error: +- if( in!=stdin ){ +- fclose(in); +- }else{ +- while( fgets(zLine, sizeof(zLine), p->in)!=0 ){ +- nLine++; +- if(strncmp(zLine, "| end ", 6)==0 ) break; ++ if( zNL ){ ++ memcpy(&zOut[iOut], ",'", 2); iOut += 2; ++ memcpy(&zOut[iOut], zNL, nNL); iOut += nNL; ++ memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12; ++ } ++ if( zCR ){ ++ memcpy(&zOut[iOut], ",'", 2); iOut += 2; ++ memcpy(&zOut[iOut], zCR, nCR); iOut += nCR; ++ memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12; ++ } ++ ++ sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT); ++ sqlite3_free(zOut); ++ return; + } +- p->lineno = nLine; + } +- sqlite3_free(a); +- utf8_printf(stderr,"Error on line %d of --hexdb input\n", nLine); +- return 0; ++ ++ sqlite3_result_value(context, argv[0]); + } +-#endif /* SQLITE_ENABLE_DESERIALIZE */ + + /* Flags for open_db(). + ** +@@ -11644,7 +12891,7 @@ + switch( p->openMode ){ + case SHELL_OPEN_APPENDVFS: { + sqlite3_open_v2(p->zDbFilename, &p->db, +- SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, "apndvfs"); ++ SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|p->openFlags, "apndvfs"); + break; + } + case SHELL_OPEN_HEXDB: +@@ -11657,12 +12904,14 @@ + break; + } + case SHELL_OPEN_READONLY: { +- sqlite3_open_v2(p->zDbFilename, &p->db, SQLITE_OPEN_READONLY, 0); ++ sqlite3_open_v2(p->zDbFilename, &p->db, ++ SQLITE_OPEN_READONLY|p->openFlags, 0); + break; + } + case SHELL_OPEN_UNSPEC: + case SHELL_OPEN_NORMAL: { +- sqlite3_open(p->zDbFilename, &p->db); ++ sqlite3_open_v2(p->zDbFilename, &p->db, ++ SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|p->openFlags, 0); + break; + } + } +@@ -11682,6 +12931,10 @@ + sqlite3_fileio_init(p->db, 0, 0); + sqlite3_shathree_init(p->db, 0, 0); + sqlite3_completion_init(p->db, 0, 0); ++ sqlite3_uint_init(p->db, 0, 0); ++#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) ++ sqlite3_dbdata_init(p->db, 0, 0); ++#endif + #ifdef SQLITE_HAVE_ZLIB + sqlite3_zipfile_init(p->db, 0, 0); + sqlite3_sqlar_init(p->db, 0, 0); +@@ -11692,6 +12945,12 @@ + shellModuleSchema, 0, 0); + sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p, + shellPutsFunc, 0, 0); ++ sqlite3_create_function(p->db, "shell_escape_crnl", 1, SQLITE_UTF8, 0, ++ shellEscapeCrnl, 0, 0); ++ sqlite3_create_function(p->db, "shell_int32", 2, SQLITE_UTF8, 0, ++ shellInt32, 0, 0); ++ sqlite3_create_function(p->db, "shell_idquote", 1, SQLITE_UTF8, 0, ++ shellIdQuote, 0, 0); + #ifndef SQLITE_NOHAVE_SYSTEM + sqlite3_create_function(p->db, "edit", 1, SQLITE_UTF8, 0, + editFunc, 0, 0); +@@ -11715,7 +12974,6 @@ + }else{ + aData = readHexDb(p, &nData); + if( aData==0 ){ +- utf8_printf(stderr, "Error in hexdb input\n"); + return; + } + } +@@ -12010,6 +13268,8 @@ + int n; /* Number of bytes in z */ + int nAlloc; /* Space allocated for z[] */ + int nLine; /* Current line number */ ++ int nRow; /* Number of rows imported */ ++ int nErr; /* Number of errors encountered */ + int bNotFirst; /* True if one or more bytes already read */ + int cTerm; /* Character that terminated the most recent field */ + int cColSep; /* The column separator character. (Usually ",") */ +@@ -12391,6 +13651,11 @@ + zCmd = sqlite3_mprintf("%s %s", zXdgOpenCmd, p->zTempFile); + if( system(zCmd) ){ + utf8_printf(stderr, "Failed: [%s]\n", zCmd); ++ }else{ ++ /* Give the start/open/xdg-open command some time to get ++ ** going before we continue, and potential delete the ++ ** p->zTempFile data file out from under it */ ++ sqlite3_sleep(2000); + } + sqlite3_free(zCmd); + outputModePop(p); +@@ -12427,7 +13692,7 @@ + } + + /* +-** Implementation of the ".info" command. ++** Implementation of the ".dbinfo" command. + ** + ** Return 1 on error, 2 to exit, and 0 otherwise. + */ +@@ -12470,12 +13735,7 @@ + "SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1", + -1, &pStmt, 0); + if( rc ){ +- if( !sqlite3_compileoption_used("ENABLE_DBPAGE_VTAB") ){ +- utf8_printf(stderr, "the \".dbinfo\" command requires the " +- "-DSQLITE_ENABLE_DBPAGE_VTAB compile-time options\n"); +- }else{ +- utf8_printf(stderr, "error: %s\n", sqlite3_errmsg(p->db)); +- } ++ utf8_printf(stderr, "error: %s\n", sqlite3_errmsg(p->db)); + sqlite3_finalize(pStmt); + return 1; + } +@@ -12684,9 +13944,21 @@ + sqlite3_file_control(p->db, 0, SQLITE_FCNTL_TEMPFILENAME, &p->zTempFile); + } + if( p->zTempFile==0 ){ ++ /* If p->db is an in-memory database then the TEMPFILENAME file-control ++ ** will not work and we will need to fallback to guessing */ ++ char *zTemp; + sqlite3_uint64 r; + sqlite3_randomness(sizeof(r), &r); +- p->zTempFile = sqlite3_mprintf("temp%llx.%s", r, zSuffix); ++ zTemp = getenv("TEMP"); ++ if( zTemp==0 ) zTemp = getenv("TMP"); ++ if( zTemp==0 ){ ++#ifdef _WIN32 ++ zTemp = "\\tmp"; ++#else ++ zTemp = "/tmp"; ++#endif ++ } ++ p->zTempFile = sqlite3_mprintf("%s/temp%llx.%s", zTemp, r, zSuffix); + }else{ + p->zTempFile = sqlite3_mprintf("%z.%s", p->zTempFile, zSuffix); + } +@@ -12946,10 +14218,7 @@ + return SQLITE_ERROR; + } + +-#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) +-/********************************************************************************* +-** The ".archive" or ".ar" command. +-*/ ++#if !defined SQLITE_OMIT_VIRTUALTABLE + static void shellPrepare( + sqlite3 *db, + int *pRc, +@@ -12968,7 +14237,14 @@ + } + } + +-static void shellPreparePrintf( ++/* ++** Create a prepared statement using printf-style arguments for the SQL. ++** ++** This routine is could be marked "static". But it is not always used, ++** depending on compile-time options. By omitting the "static", we avoid ++** nuisance compiler warnings about "defined but not used". ++*/ ++void shellPreparePrintf( + sqlite3 *db, + int *pRc, + sqlite3_stmt **ppStmt, +@@ -12991,7 +14267,13 @@ + } + } + +-static void shellFinalize( ++/* Finalize the prepared statement created using shellPreparePrintf(). ++** ++** This routine is could be marked "static". But it is not always used, ++** depending on compile-time options. By omitting the "static", we avoid ++** nuisance compiler warnings about "defined but not used". ++*/ ++void shellFinalize( + int *pRc, + sqlite3_stmt *pStmt + ){ +@@ -13007,7 +14289,13 @@ + } + } + +-static void shellReset( ++/* Reset the prepared statement created using shellPreparePrintf(). ++** ++** This routine is could be marked "static". But it is not always used, ++** depending on compile-time options. By omitting the "static", we avoid ++** nuisance compiler warnings about "defined but not used". ++*/ ++void shellReset( + int *pRc, + sqlite3_stmt *pStmt + ){ +@@ -13020,6 +14308,12 @@ + *pRc = rc; + } + } ++#endif /* !defined SQLITE_OMIT_VIRTUALTABLE */ ++ ++#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) ++/****************************************************************************** ++** The ".archive" or ".ar" command. ++*/ + /* + ** Structure representing a single ".ar" command. + */ +@@ -13215,7 +14509,8 @@ + i = n; + }else{ + if( iArg>=(nArg-1) ){ +- return arErrorMsg(pAr, "option requires an argument: %c",z[i]); ++ return arErrorMsg(pAr, "option requires an argument: %c", ++ z[i]); + } + zArg = azArg[++iArg]; + } +@@ -13603,10 +14898,10 @@ + ** Implementation of ".ar" dot command. + */ + static int arDotCommand( +- ShellState *pState, /* Current shell tool state */ +- int fromCmdLine, /* True if -A command-line option, not .ar cmd */ +- char **azArg, /* Array of arguments passed to dot command */ +- int nArg /* Number of entries in azArg[] */ ++ ShellState *pState, /* Current shell tool state */ ++ int fromCmdLine, /* True if -A command-line option, not .ar cmd */ ++ char **azArg, /* Array of arguments passed to dot command */ ++ int nArg /* Number of entries in azArg[] */ + ){ + ArCommand cmd; + int rc; +@@ -13691,23 +14986,698 @@ + rc = arCreateOrUpdateCommand(&cmd, 1, 0); + break; + +- default: +- assert( cmd.eCmd==AR_CMD_UPDATE ); +- rc = arCreateOrUpdateCommand(&cmd, 1, 1); +- break; ++ default: ++ assert( cmd.eCmd==AR_CMD_UPDATE ); ++ rc = arCreateOrUpdateCommand(&cmd, 1, 1); ++ break; ++ } ++ } ++end_ar_command: ++ if( cmd.db!=pState->db ){ ++ close_db(cmd.db); ++ } ++ sqlite3_free(cmd.zSrcTable); ++ ++ return rc; ++} ++/* End of the ".archive" or ".ar" command logic ++*******************************************************************************/ ++#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */ ++ ++#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) ++/* ++** If (*pRc) is not SQLITE_OK when this function is called, it is a no-op. ++** Otherwise, the SQL statement or statements in zSql are executed using ++** database connection db and the error code written to *pRc before ++** this function returns. ++*/ ++static void shellExec(sqlite3 *db, int *pRc, const char *zSql){ ++ int rc = *pRc; ++ if( rc==SQLITE_OK ){ ++ char *zErr = 0; ++ rc = sqlite3_exec(db, zSql, 0, 0, &zErr); ++ if( rc!=SQLITE_OK ){ ++ raw_printf(stderr, "SQL error: %s\n", zErr); ++ } ++ *pRc = rc; ++ } ++} ++ ++/* ++** Like shellExec(), except that zFmt is a printf() style format string. ++*/ ++static void shellExecPrintf(sqlite3 *db, int *pRc, const char *zFmt, ...){ ++ char *z = 0; ++ if( *pRc==SQLITE_OK ){ ++ va_list ap; ++ va_start(ap, zFmt); ++ z = sqlite3_vmprintf(zFmt, ap); ++ va_end(ap); ++ if( z==0 ){ ++ *pRc = SQLITE_NOMEM; ++ }else{ ++ shellExec(db, pRc, z); ++ } ++ sqlite3_free(z); ++ } ++} ++ ++/* ++** If *pRc is not SQLITE_OK when this function is called, it is a no-op. ++** Otherwise, an attempt is made to allocate, zero and return a pointer ++** to a buffer nByte bytes in size. If an OOM error occurs, *pRc is set ++** to SQLITE_NOMEM and NULL returned. ++*/ ++static void *shellMalloc(int *pRc, sqlite3_int64 nByte){ ++ void *pRet = 0; ++ if( *pRc==SQLITE_OK ){ ++ pRet = sqlite3_malloc64(nByte); ++ if( pRet==0 ){ ++ *pRc = SQLITE_NOMEM; ++ }else{ ++ memset(pRet, 0, nByte); ++ } ++ } ++ return pRet; ++} ++ ++/* ++** If *pRc is not SQLITE_OK when this function is called, it is a no-op. ++** Otherwise, zFmt is treated as a printf() style string. The result of ++** formatting it along with any trailing arguments is written into a ++** buffer obtained from sqlite3_malloc(), and pointer to which is returned. ++** It is the responsibility of the caller to eventually free this buffer ++** using a call to sqlite3_free(). ++** ++** If an OOM error occurs, (*pRc) is set to SQLITE_NOMEM and a NULL ++** pointer returned. ++*/ ++static char *shellMPrintf(int *pRc, const char *zFmt, ...){ ++ char *z = 0; ++ if( *pRc==SQLITE_OK ){ ++ va_list ap; ++ va_start(ap, zFmt); ++ z = sqlite3_vmprintf(zFmt, ap); ++ va_end(ap); ++ if( z==0 ){ ++ *pRc = SQLITE_NOMEM; ++ } ++ } ++ return z; ++} ++ ++/* ++** When running the ".recover" command, each output table, and the special ++** orphaned row table if it is required, is represented by an instance ++** of the following struct. ++*/ ++typedef struct RecoverTable RecoverTable; ++struct RecoverTable { ++ char *zQuoted; /* Quoted version of table name */ ++ int nCol; /* Number of columns in table */ ++ char **azlCol; /* Array of column lists */ ++ int iPk; /* Index of IPK column */ ++}; ++ ++/* ++** Free a RecoverTable object allocated by recoverFindTable() or ++** recoverOrphanTable(). ++*/ ++static void recoverFreeTable(RecoverTable *pTab){ ++ if( pTab ){ ++ sqlite3_free(pTab->zQuoted); ++ if( pTab->azlCol ){ ++ int i; ++ for(i=0; i<=pTab->nCol; i++){ ++ sqlite3_free(pTab->azlCol[i]); ++ } ++ sqlite3_free(pTab->azlCol); ++ } ++ sqlite3_free(pTab); ++ } ++} ++ ++/* ++** This function is a no-op if (*pRc) is not SQLITE_OK when it is called. ++** Otherwise, it allocates and returns a RecoverTable object based on the ++** final four arguments passed to this function. It is the responsibility ++** of the caller to eventually free the returned object using ++** recoverFreeTable(). ++*/ ++static RecoverTable *recoverNewTable( ++ int *pRc, /* IN/OUT: Error code */ ++ const char *zName, /* Name of table */ ++ const char *zSql, /* CREATE TABLE statement */ ++ int bIntkey, ++ int nCol ++){ ++ sqlite3 *dbtmp = 0; /* sqlite3 handle for testing CREATE TABLE */ ++ int rc = *pRc; ++ RecoverTable *pTab = 0; ++ ++ pTab = (RecoverTable*)shellMalloc(&rc, sizeof(RecoverTable)); ++ if( rc==SQLITE_OK ){ ++ int nSqlCol = 0; ++ int bSqlIntkey = 0; ++ sqlite3_stmt *pStmt = 0; ++ ++ rc = sqlite3_open("", &dbtmp); ++ if( rc==SQLITE_OK ){ ++ sqlite3_create_function(dbtmp, "shell_idquote", 1, SQLITE_UTF8, 0, ++ shellIdQuote, 0, 0); ++ } ++ if( rc==SQLITE_OK ){ ++ rc = sqlite3_exec(dbtmp, "PRAGMA writable_schema = on", 0, 0, 0); ++ } ++ if( rc==SQLITE_OK ){ ++ rc = sqlite3_exec(dbtmp, zSql, 0, 0, 0); ++ if( rc==SQLITE_ERROR ){ ++ rc = SQLITE_OK; ++ goto finished; ++ } ++ } ++ shellPreparePrintf(dbtmp, &rc, &pStmt, ++ "SELECT count(*) FROM pragma_table_info(%Q)", zName ++ ); ++ if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ ++ nSqlCol = sqlite3_column_int(pStmt, 0); ++ } ++ shellFinalize(&rc, pStmt); ++ ++ if( rc!=SQLITE_OK || nSqlCol<nCol ){ ++ goto finished; ++ } ++ ++ shellPreparePrintf(dbtmp, &rc, &pStmt, ++ "SELECT (" ++ " SELECT substr(data,1,1)==X'0D' FROM sqlite_dbpage WHERE pgno=rootpage" ++ ") FROM sqlite_master WHERE name = %Q", zName ++ ); ++ if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ ++ bSqlIntkey = sqlite3_column_int(pStmt, 0); ++ } ++ shellFinalize(&rc, pStmt); ++ ++ if( bIntkey==bSqlIntkey ){ ++ int i; ++ const char *zPk = "_rowid_"; ++ sqlite3_stmt *pPkFinder = 0; ++ ++ /* If this is an intkey table and there is an INTEGER PRIMARY KEY, ++ ** set zPk to the name of the PK column, and pTab->iPk to the index ++ ** of the column, where columns are 0-numbered from left to right. ++ ** Or, if this is a WITHOUT ROWID table or if there is no IPK column, ++ ** leave zPk as "_rowid_" and pTab->iPk at -2. */ ++ pTab->iPk = -2; ++ if( bIntkey ){ ++ shellPreparePrintf(dbtmp, &rc, &pPkFinder, ++ "SELECT cid, name FROM pragma_table_info(%Q) " ++ " WHERE pk=1 AND type='integer' COLLATE nocase" ++ " AND NOT EXISTS (SELECT cid FROM pragma_table_info(%Q) WHERE pk=2)" ++ , zName, zName ++ ); ++ if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPkFinder) ){ ++ pTab->iPk = sqlite3_column_int(pPkFinder, 0); ++ zPk = (const char*)sqlite3_column_text(pPkFinder, 1); ++ } ++ } ++ ++ pTab->zQuoted = shellMPrintf(&rc, "\"%w\"", zName); ++ pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * (nSqlCol+1)); ++ pTab->nCol = nSqlCol; ++ ++ if( bIntkey ){ ++ pTab->azlCol[0] = shellMPrintf(&rc, "\"%w\"", zPk); ++ }else{ ++ pTab->azlCol[0] = shellMPrintf(&rc, ""); ++ } ++ i = 1; ++ shellPreparePrintf(dbtmp, &rc, &pStmt, ++ "SELECT %Q || group_concat(shell_idquote(name), ', ') " ++ " FILTER (WHERE cid!=%d) OVER (ORDER BY %s cid) " ++ "FROM pragma_table_info(%Q)", ++ bIntkey ? ", " : "", pTab->iPk, ++ bIntkey ? "" : "(CASE WHEN pk=0 THEN 1000000 ELSE pk END), ", ++ zName ++ ); ++ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ ++ const char *zText = (const char*)sqlite3_column_text(pStmt, 0); ++ pTab->azlCol[i] = shellMPrintf(&rc, "%s%s", pTab->azlCol[0], zText); ++ i++; ++ } ++ shellFinalize(&rc, pStmt); ++ ++ shellFinalize(&rc, pPkFinder); ++ } ++ } ++ ++ finished: ++ sqlite3_close(dbtmp); ++ *pRc = rc; ++ if( rc!=SQLITE_OK || (pTab && pTab->zQuoted==0) ){ ++ recoverFreeTable(pTab); ++ pTab = 0; ++ } ++ return pTab; ++} ++ ++/* ++** This function is called to search the schema recovered from the ++** sqlite_master table of the (possibly) corrupt database as part ++** of a ".recover" command. Specifically, for a table with root page ++** iRoot and at least nCol columns. Additionally, if bIntkey is 0, the ++** table must be a WITHOUT ROWID table, or if non-zero, not one of ++** those. ++** ++** If a table is found, a (RecoverTable*) object is returned. Or, if ++** no such table is found, but bIntkey is false and iRoot is the ++** root page of an index in the recovered schema, then (*pbNoop) is ++** set to true and NULL returned. Or, if there is no such table or ++** index, NULL is returned and (*pbNoop) set to 0, indicating that ++** the caller should write data to the orphans table. ++*/ ++static RecoverTable *recoverFindTable( ++ ShellState *pState, /* Shell state object */ ++ int *pRc, /* IN/OUT: Error code */ ++ int iRoot, /* Root page of table */ ++ int bIntkey, /* True for an intkey table */ ++ int nCol, /* Number of columns in table */ ++ int *pbNoop /* OUT: True if iRoot is root of index */ ++){ ++ sqlite3_stmt *pStmt = 0; ++ RecoverTable *pRet = 0; ++ int bNoop = 0; ++ const char *zSql = 0; ++ const char *zName = 0; ++ ++ /* Search the recovered schema for an object with root page iRoot. */ ++ shellPreparePrintf(pState->db, pRc, &pStmt, ++ "SELECT type, name, sql FROM recovery.schema WHERE rootpage=%d", iRoot ++ ); ++ while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ ++ const char *zType = (const char*)sqlite3_column_text(pStmt, 0); ++ if( bIntkey==0 && sqlite3_stricmp(zType, "index")==0 ){ ++ bNoop = 1; ++ break; ++ } ++ if( sqlite3_stricmp(zType, "table")==0 ){ ++ zName = (const char*)sqlite3_column_text(pStmt, 1); ++ zSql = (const char*)sqlite3_column_text(pStmt, 2); ++ pRet = recoverNewTable(pRc, zName, zSql, bIntkey, nCol); ++ break; ++ } ++ } ++ ++ shellFinalize(pRc, pStmt); ++ *pbNoop = bNoop; ++ return pRet; ++} ++ ++/* ++** Return a RecoverTable object representing the orphans table. ++*/ ++static RecoverTable *recoverOrphanTable( ++ ShellState *pState, /* Shell state object */ ++ int *pRc, /* IN/OUT: Error code */ ++ const char *zLostAndFound, /* Base name for orphans table */ ++ int nCol /* Number of user data columns */ ++){ ++ RecoverTable *pTab = 0; ++ if( nCol>=0 && *pRc==SQLITE_OK ){ ++ int i; ++ ++ /* This block determines the name of the orphan table. The prefered ++ ** name is zLostAndFound. But if that clashes with another name ++ ** in the recovered schema, try zLostAndFound_0, zLostAndFound_1 ++ ** and so on until a non-clashing name is found. */ ++ int iTab = 0; ++ char *zTab = shellMPrintf(pRc, "%s", zLostAndFound); ++ sqlite3_stmt *pTest = 0; ++ shellPrepare(pState->db, pRc, ++ "SELECT 1 FROM recovery.schema WHERE name=?", &pTest ++ ); ++ if( pTest ) sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); ++ while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pTest) ){ ++ shellReset(pRc, pTest); ++ sqlite3_free(zTab); ++ zTab = shellMPrintf(pRc, "%s_%d", zLostAndFound, iTab++); ++ sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); ++ } ++ shellFinalize(pRc, pTest); ++ ++ pTab = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); ++ if( pTab ){ ++ pTab->zQuoted = shellMPrintf(pRc, "\"%w\"", zTab); ++ pTab->nCol = nCol; ++ pTab->iPk = -2; ++ if( nCol>0 ){ ++ pTab->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * (nCol+1)); ++ if( pTab->azlCol ){ ++ pTab->azlCol[nCol] = shellMPrintf(pRc, ""); ++ for(i=nCol-1; i>=0; i--){ ++ pTab->azlCol[i] = shellMPrintf(pRc, "%s, NULL", pTab->azlCol[i+1]); ++ } ++ } ++ } ++ ++ if( *pRc!=SQLITE_OK ){ ++ recoverFreeTable(pTab); ++ pTab = 0; ++ }else{ ++ raw_printf(pState->out, ++ "CREATE TABLE %s(rootpgno INTEGER, " ++ "pgno INTEGER, nfield INTEGER, id INTEGER", pTab->zQuoted ++ ); ++ for(i=0; i<nCol; i++){ ++ raw_printf(pState->out, ", c%d", i); ++ } ++ raw_printf(pState->out, ");\n"); ++ } ++ } ++ sqlite3_free(zTab); ++ } ++ return pTab; ++} ++ ++/* ++** This function is called to recover data from the database. A script ++** to construct a new database containing all recovered data is output ++** on stream pState->out. ++*/ ++static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ ++ int rc = SQLITE_OK; ++ sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ ++ sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */ ++ sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */ ++ const char *zRecoveryDb = ""; /* Name of "recovery" database */ ++ const char *zLostAndFound = "lost_and_found"; ++ int i; ++ int nOrphan = -1; ++ RecoverTable *pOrphan = 0; ++ ++ int bFreelist = 1; /* 0 if --freelist-corrupt is specified */ ++ int bRowids = 1; /* 0 if --no-rowids */ ++ for(i=1; i<nArg; i++){ ++ char *z = azArg[i]; ++ int n; ++ if( z[0]=='-' && z[1]=='-' ) z++; ++ n = strlen30(z); ++ if( n<=17 && memcmp("-freelist-corrupt", z, n)==0 ){ ++ bFreelist = 0; ++ }else ++ if( n<=12 && memcmp("-recovery-db", z, n)==0 && i<(nArg-1) ){ ++ i++; ++ zRecoveryDb = azArg[i]; ++ }else ++ if( n<=15 && memcmp("-lost-and-found", z, n)==0 && i<(nArg-1) ){ ++ i++; ++ zLostAndFound = azArg[i]; ++ }else ++ if( n<=10 && memcmp("-no-rowids", z, n)==0 ){ ++ bRowids = 0; ++ } ++ else{ ++ utf8_printf(stderr, "unexpected option: %s\n", azArg[i]); ++ showHelp(pState->out, azArg[0]); ++ return 1; ++ } ++ } ++ ++ shellExecPrintf(pState->db, &rc, ++ /* Attach an in-memory database named 'recovery'. Create an indexed ++ ** cache of the sqlite_dbptr virtual table. */ ++ "PRAGMA writable_schema = on;" ++ "ATTACH %Q AS recovery;" ++ "DROP TABLE IF EXISTS recovery.dbptr;" ++ "DROP TABLE IF EXISTS recovery.freelist;" ++ "DROP TABLE IF EXISTS recovery.map;" ++ "DROP TABLE IF EXISTS recovery.schema;" ++ "CREATE TABLE recovery.freelist(pgno INTEGER PRIMARY KEY);", zRecoveryDb ++ ); ++ ++ if( bFreelist ){ ++ shellExec(pState->db, &rc, ++ "WITH trunk(pgno) AS (" ++ " SELECT shell_int32(" ++ " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 8) AS x " ++ " WHERE x>0" ++ " UNION" ++ " SELECT shell_int32(" ++ " (SELECT data FROM sqlite_dbpage WHERE pgno=trunk.pgno), 0) AS x " ++ " FROM trunk WHERE x>0" ++ ")," ++ "freelist(data, n, freepgno) AS (" ++ " SELECT data, min(16384, shell_int32(data, 1)-1), t.pgno " ++ " FROM trunk t, sqlite_dbpage s WHERE s.pgno=t.pgno" ++ " UNION ALL" ++ " SELECT data, n-1, shell_int32(data, 2+n) " ++ " FROM freelist WHERE n>=0" ++ ")" ++ "REPLACE INTO recovery.freelist SELECT freepgno FROM freelist;" ++ ); ++ } ++ ++ /* If this is an auto-vacuum database, add all pointer-map pages to ++ ** the freelist table. Do this regardless of whether or not ++ ** --freelist-corrupt was specified. */ ++ shellExec(pState->db, &rc, ++ "WITH ptrmap(pgno) AS (" ++ " SELECT 2 WHERE shell_int32(" ++ " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 13" ++ " )" ++ " UNION ALL " ++ " SELECT pgno+1+(SELECT page_size FROM pragma_page_size)/5 AS pp " ++ " FROM ptrmap WHERE pp<=(SELECT page_count FROM pragma_page_count)" ++ ")" ++ "REPLACE INTO recovery.freelist SELECT pgno FROM ptrmap" ++ ); ++ ++ shellExec(pState->db, &rc, ++ "CREATE TABLE recovery.dbptr(" ++ " pgno, child, PRIMARY KEY(child, pgno)" ++ ") WITHOUT ROWID;" ++ "INSERT OR IGNORE INTO recovery.dbptr(pgno, child) " ++ " SELECT * FROM sqlite_dbptr" ++ " WHERE pgno NOT IN freelist AND child NOT IN freelist;" ++ ++ /* Delete any pointer to page 1. This ensures that page 1 is considered ++ ** a root page, regardless of how corrupt the db is. */ ++ "DELETE FROM recovery.dbptr WHERE child = 1;" ++ ++ /* Delete all pointers to any pages that have more than one pointer ++ ** to them. Such pages will be treated as root pages when recovering ++ ** data. */ ++ "DELETE FROM recovery.dbptr WHERE child IN (" ++ " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1" ++ ");" ++ ++ /* Create the "map" table that will (eventually) contain instructions ++ ** for dealing with each page in the db that contains one or more ++ ** records. */ ++ "CREATE TABLE recovery.map(" ++ "pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT" ++ ");" ++ ++ /* Populate table [map]. If there are circular loops of pages in the ++ ** database, the following adds all pages in such a loop to the map ++ ** as individual root pages. This could be handled better. */ ++ "WITH pages(i, maxlen) AS (" ++ " SELECT page_count, (" ++ " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=page_count" ++ " ) FROM pragma_page_count WHERE page_count>0" ++ " UNION ALL" ++ " SELECT i-1, (" ++ " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=i-1" ++ " ) FROM pages WHERE i>=2" ++ ")" ++ "INSERT INTO recovery.map(pgno, maxlen, intkey, root) " ++ " SELECT i, maxlen, NULL, (" ++ " WITH p(orig, pgno, parent) AS (" ++ " SELECT 0, i, (SELECT pgno FROM recovery.dbptr WHERE child=i)" ++ " UNION " ++ " SELECT i, p.parent, " ++ " (SELECT pgno FROM recovery.dbptr WHERE child=p.parent) FROM p" ++ " )" ++ " SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)" ++ ") " ++ "FROM pages WHERE maxlen IS NOT NULL AND i NOT IN freelist;" ++ "UPDATE recovery.map AS o SET intkey = (" ++ " SELECT substr(data, 1, 1)==X'0D' FROM sqlite_dbpage WHERE pgno=o.pgno" ++ ");" ++ ++ /* Extract data from page 1 and any linked pages into table ++ ** recovery.schema. With the same schema as an sqlite_master table. */ ++ "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" ++ "INSERT INTO recovery.schema SELECT " ++ " max(CASE WHEN field=0 THEN value ELSE NULL END)," ++ " max(CASE WHEN field=1 THEN value ELSE NULL END)," ++ " max(CASE WHEN field=2 THEN value ELSE NULL END)," ++ " max(CASE WHEN field=3 THEN value ELSE NULL END)," ++ " max(CASE WHEN field=4 THEN value ELSE NULL END)" ++ "FROM sqlite_dbdata WHERE pgno IN (" ++ " SELECT pgno FROM recovery.map WHERE root=1" ++ ")" ++ "GROUP BY pgno, cell;" ++ "CREATE INDEX recovery.schema_rootpage ON schema(rootpage);" ++ ); ++ ++ /* Open a transaction, then print out all non-virtual, non-"sqlite_%" ++ ** CREATE TABLE statements that extracted from the existing schema. */ ++ if( rc==SQLITE_OK ){ ++ sqlite3_stmt *pStmt = 0; ++ /* ".recover" might output content in an order which causes immediate ++ ** foreign key constraints to be violated. So disable foreign-key ++ ** constraint enforcement to prevent problems when running the output ++ ** script. */ ++ raw_printf(pState->out, "PRAGMA foreign_keys=OFF;\n"); ++ raw_printf(pState->out, "BEGIN;\n"); ++ raw_printf(pState->out, "PRAGMA writable_schema = on;\n"); ++ shellPrepare(pState->db, &rc, ++ "SELECT sql FROM recovery.schema " ++ "WHERE type='table' AND sql LIKE 'create table%'", &pStmt ++ ); ++ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ ++ const char *zCreateTable = (const char*)sqlite3_column_text(pStmt, 0); ++ raw_printf(pState->out, "CREATE TABLE IF NOT EXISTS %s;\n", ++ &zCreateTable[12] ++ ); ++ } ++ shellFinalize(&rc, pStmt); ++ } ++ ++ /* Figure out if an orphan table will be required. And if so, how many ++ ** user columns it should contain */ ++ shellPrepare(pState->db, &rc, ++ "SELECT coalesce(max(maxlen), -2) FROM recovery.map WHERE root>1" ++ , &pLoop ++ ); ++ if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ ++ nOrphan = sqlite3_column_int(pLoop, 0); ++ } ++ shellFinalize(&rc, pLoop); ++ pLoop = 0; ++ ++ shellPrepare(pState->db, &rc, ++ "SELECT pgno FROM recovery.map WHERE root=?", &pPages ++ ); ++ ++ shellPrepare(pState->db, &rc, ++ "SELECT max(field), group_concat(shell_escape_crnl(quote" ++ "(case when (? AND field<0) then NULL else value end)" ++ "), ', ')" ++ ", min(field) " ++ "FROM sqlite_dbdata WHERE pgno = ? AND field != ?" ++ "GROUP BY cell", &pCells ++ ); ++ ++ /* Loop through each root page. */ ++ shellPrepare(pState->db, &rc, ++ "SELECT root, intkey, max(maxlen) FROM recovery.map" ++ " WHERE root>1 GROUP BY root, intkey ORDER BY root=(" ++ " SELECT rootpage FROM recovery.schema WHERE name='sqlite_sequence'" ++ ")", &pLoop ++ ); ++ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ ++ int iRoot = sqlite3_column_int(pLoop, 0); ++ int bIntkey = sqlite3_column_int(pLoop, 1); ++ int nCol = sqlite3_column_int(pLoop, 2); ++ int bNoop = 0; ++ RecoverTable *pTab; ++ ++ assert( bIntkey==0 || bIntkey==1 ); ++ pTab = recoverFindTable(pState, &rc, iRoot, bIntkey, nCol, &bNoop); ++ if( bNoop || rc ) continue; ++ if( pTab==0 ){ ++ if( pOrphan==0 ){ ++ pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan); ++ } ++ pTab = pOrphan; ++ if( pTab==0 ) break; ++ } ++ ++ if( 0==sqlite3_stricmp(pTab->zQuoted, "\"sqlite_sequence\"") ){ ++ raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); ++ } ++ sqlite3_bind_int(pPages, 1, iRoot); ++ if( bRowids==0 && pTab->iPk<0 ){ ++ sqlite3_bind_int(pCells, 1, 1); ++ }else{ ++ sqlite3_bind_int(pCells, 1, 0); ++ } ++ sqlite3_bind_int(pCells, 3, pTab->iPk); ++ ++ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPages) ){ ++ int iPgno = sqlite3_column_int(pPages, 0); ++ sqlite3_bind_int(pCells, 2, iPgno); ++ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCells) ){ ++ int nField = sqlite3_column_int(pCells, 0); ++ int iMin = sqlite3_column_int(pCells, 2); ++ const char *zVal = (const char*)sqlite3_column_text(pCells, 1); ++ ++ RecoverTable *pTab2 = pTab; ++ if( pTab!=pOrphan && (iMin<0)!=bIntkey ){ ++ if( pOrphan==0 ){ ++ pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan); ++ } ++ pTab2 = pOrphan; ++ if( pTab2==0 ) break; ++ } ++ ++ nField = nField+1; ++ if( pTab2==pOrphan ){ ++ raw_printf(pState->out, ++ "INSERT INTO %s VALUES(%d, %d, %d, %s%s%s);\n", ++ pTab2->zQuoted, iRoot, iPgno, nField, ++ iMin<0 ? "" : "NULL, ", zVal, pTab2->azlCol[nField] ++ ); ++ }else{ ++ raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", ++ pTab2->zQuoted, pTab2->azlCol[nField], zVal ++ ); ++ } ++ } ++ shellReset(&rc, pCells); + } ++ shellReset(&rc, pPages); ++ if( pTab!=pOrphan ) recoverFreeTable(pTab); + } +-end_ar_command: +- if( cmd.db!=pState->db ){ +- close_db(cmd.db); ++ shellFinalize(&rc, pLoop); ++ shellFinalize(&rc, pPages); ++ shellFinalize(&rc, pCells); ++ recoverFreeTable(pOrphan); ++ ++ /* The rest of the schema */ ++ if( rc==SQLITE_OK ){ ++ sqlite3_stmt *pStmt = 0; ++ shellPrepare(pState->db, &rc, ++ "SELECT sql, name FROM recovery.schema " ++ "WHERE sql NOT LIKE 'create table%'", &pStmt ++ ); ++ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ ++ const char *zSql = (const char*)sqlite3_column_text(pStmt, 0); ++ if( sqlite3_strnicmp(zSql, "create virt", 11)==0 ){ ++ const char *zName = (const char*)sqlite3_column_text(pStmt, 1); ++ char *zPrint = shellMPrintf(&rc, ++ "INSERT INTO sqlite_master VALUES('table', %Q, %Q, 0, %Q)", ++ zName, zName, zSql ++ ); ++ raw_printf(pState->out, "%s;\n", zPrint); ++ sqlite3_free(zPrint); ++ }else{ ++ raw_printf(pState->out, "%s;\n", zSql); ++ } ++ } ++ shellFinalize(&rc, pStmt); + } +- sqlite3_free(cmd.zSrcTable); + ++ if( rc==SQLITE_OK ){ ++ raw_printf(pState->out, "PRAGMA writable_schema = off;\n"); ++ raw_printf(pState->out, "COMMIT;\n"); ++ } ++ sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0); + return rc; + } +-/* End of the ".archive" or ".ar" command logic +-**********************************************************************************/ +-#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */ ++#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ + + + /* +@@ -13721,7 +15691,7 @@ + int nArg = 0; + int n, c; + int rc = 0; +- char *azArg[50]; ++ char *azArg[52]; + + #ifndef SQLITE_OMIT_VIRTUALTABLE + if( p->expert.pExpert ){ +@@ -13731,7 +15701,7 @@ + + /* Parse the input line into tokens. + */ +- while( zLine[h] && nArg<ArraySize(azArg) ){ ++ while( zLine[h] && nArg<ArraySize(azArg)-1 ){ + while( IsSpace(zLine[h]) ){ h++; } + if( zLine[h]==0 ) break; + if( zLine[h]=='\'' || zLine[h]=='"' ){ +@@ -13752,6 +15722,7 @@ + resolve_backslashes(azArg[nArg-1]); + } + } ++ azArg[nArg] = 0; + + /* Process the input line. + */ +@@ -13965,15 +15936,22 @@ + const char *zName; + int op; + } aDbConfig[] = { +- { "enable_fkey", SQLITE_DBCONFIG_ENABLE_FKEY }, +- { "enable_trigger", SQLITE_DBCONFIG_ENABLE_TRIGGER }, +- { "fts3_tokenizer", SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER }, +- { "load_extension", SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION }, +- { "no_ckpt_on_close", SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE }, +- { "enable_qpsg", SQLITE_DBCONFIG_ENABLE_QPSG }, +- { "trigger_eqp", SQLITE_DBCONFIG_TRIGGER_EQP }, +- { "reset_database", SQLITE_DBCONFIG_RESET_DATABASE }, +- { "defensive", SQLITE_DBCONFIG_DEFENSIVE }, ++ { "defensive", SQLITE_DBCONFIG_DEFENSIVE }, ++ { "dqs_ddl", SQLITE_DBCONFIG_DQS_DDL }, ++ { "dqs_dml", SQLITE_DBCONFIG_DQS_DML }, ++ { "enable_fkey", SQLITE_DBCONFIG_ENABLE_FKEY }, ++ { "enable_qpsg", SQLITE_DBCONFIG_ENABLE_QPSG }, ++ { "enable_trigger", SQLITE_DBCONFIG_ENABLE_TRIGGER }, ++ { "enable_view", SQLITE_DBCONFIG_ENABLE_VIEW }, ++ { "fts3_tokenizer", SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER }, ++ { "legacy_alter_table", SQLITE_DBCONFIG_LEGACY_ALTER_TABLE }, ++ { "legacy_file_format", SQLITE_DBCONFIG_LEGACY_FILE_FORMAT }, ++ { "load_extension", SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION }, ++ { "no_ckpt_on_close", SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE }, ++ { "reset_database", SQLITE_DBCONFIG_RESET_DATABASE }, ++ { "trigger_eqp", SQLITE_DBCONFIG_TRIGGER_EQP }, ++ { "trusted_schema", SQLITE_DBCONFIG_TRUSTED_SCHEMA }, ++ { "writable_schema", SQLITE_DBCONFIG_WRITABLE_SCHEMA }, + }; + int ii, v; + open_db(p, 0); +@@ -13983,7 +15961,7 @@ + sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0); + } + sqlite3_db_config(p->db, aDbConfig[ii].op, -1, &v); +- utf8_printf(p->out, "%18s %s\n", aDbConfig[ii].zName, v ? "on" : "off"); ++ utf8_printf(p->out, "%19s %s\n", aDbConfig[ii].zName, v ? "on" : "off"); + if( nArg>1 ) break; + } + if( nArg>1 && ii==ArraySize(aDbConfig) ){ +@@ -13996,8 +15974,16 @@ + rc = shell_dbinfo_command(p, nArg, azArg); + }else + ++#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) ++ if( c=='r' && strncmp(azArg[0], "recover", n)==0 ){ ++ open_db(p, 0); ++ rc = recoverDatabaseCmd(p, nArg, azArg); ++ }else ++#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ ++ + if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ +- const char *zLike = 0; ++ char *zLike = 0; ++ char *zSql; + int i; + int savedShowHeader = p->showHeader; + int savedShellFlags = p->shellFlgs; +@@ -14025,15 +16011,15 @@ + goto meta_command_exit; + } + }else if( zLike ){ +- raw_printf(stderr, "Usage: .dump ?--preserve-rowids? " +- "?--newlines? ?LIKE-PATTERN?\n"); +- rc = 1; +- goto meta_command_exit; ++ zLike = sqlite3_mprintf("%z OR name LIKE %Q ESCAPE '\\'", ++ zLike, azArg[i]); + }else{ +- zLike = azArg[i]; ++ zLike = sqlite3_mprintf("name LIKE %Q ESCAPE '\\'", azArg[i]); + } + } ++ + open_db(p, 0); ++ + /* When playing back a "dump", the content might appear in an order + ** which causes immediate foreign key constraints to be violated. + ** So disable foreign-key constraint enforcement to prevent problems. */ +@@ -14046,42 +16032,32 @@ + ** corrupt. */ + sqlite3_exec(p->db, "SAVEPOINT dump; PRAGMA writable_schema=ON", 0, 0, 0); + p->nErr = 0; +- if( zLike==0 ){ +- run_schema_dump_query(p, +- "SELECT name, type, sql FROM sqlite_master " +- "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'" +- ); +- run_schema_dump_query(p, +- "SELECT name, type, sql FROM sqlite_master " +- "WHERE name=='sqlite_sequence'" +- ); +- run_table_dump_query(p, +- "SELECT sql FROM sqlite_master " +- "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 +- ); +- }else{ +- char *zSql; +- zSql = sqlite3_mprintf( +- "SELECT name, type, sql FROM sqlite_master " +- "WHERE tbl_name LIKE %Q AND type=='table'" +- " AND sql NOT NULL", zLike); +- run_schema_dump_query(p,zSql); +- sqlite3_free(zSql); +- zSql = sqlite3_mprintf( +- "SELECT sql FROM sqlite_master " +- "WHERE sql NOT NULL" +- " AND type IN ('index','trigger','view')" +- " AND tbl_name LIKE %Q", zLike); +- run_table_dump_query(p, zSql, 0); +- sqlite3_free(zSql); +- } ++ if( zLike==0 ) zLike = sqlite3_mprintf("true"); ++ zSql = sqlite3_mprintf( ++ "SELECT name, type, sql FROM sqlite_master " ++ "WHERE (%s) AND type=='table'" ++ " AND sql NOT NULL" ++ " ORDER BY tbl_name='sqlite_sequence', rowid", ++ zLike ++ ); ++ run_schema_dump_query(p,zSql); ++ sqlite3_free(zSql); ++ zSql = sqlite3_mprintf( ++ "SELECT sql FROM sqlite_master " ++ "WHERE (%s) AND sql NOT NULL" ++ " AND type IN ('index','trigger','view')", ++ zLike ++ ); ++ run_table_dump_query(p, zSql); ++ sqlite3_free(zSql); ++ sqlite3_free(zLike); + if( p->writableSchema ){ + raw_printf(p->out, "PRAGMA writable_schema=OFF;\n"); + p->writableSchema = 0; + } + sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0); + sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0); +- raw_printf(p->out, p->nErr ? "ROLLBACK; -- due to errors\n" : "COMMIT;\n"); ++ raw_printf(p->out, p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n"); + p->showHeader = savedShowHeader; + p->shellFlgs = savedShellFlags; + }else +@@ -14162,6 +16138,151 @@ + }else + #endif + ++ if( c=='f' && strncmp(azArg[0], "filectrl", n)==0 ){ ++ static const struct { ++ const char *zCtrlName; /* Name of a test-control option */ ++ int ctrlCode; /* Integer code for that option */ ++ const char *zUsage; /* Usage notes */ ++ } aCtrl[] = { ++ { "size_limit", SQLITE_FCNTL_SIZE_LIMIT, "[LIMIT]" }, ++ { "chunk_size", SQLITE_FCNTL_CHUNK_SIZE, "SIZE" }, ++ /* { "win32_av_retry", SQLITE_FCNTL_WIN32_AV_RETRY, "COUNT DELAY" },*/ ++ { "persist_wal", SQLITE_FCNTL_PERSIST_WAL, "[BOOLEAN]" }, ++ { "psow", SQLITE_FCNTL_POWERSAFE_OVERWRITE, "[BOOLEAN]" }, ++ /* { "pragma", SQLITE_FCNTL_PRAGMA, "NAME ARG" },*/ ++ { "tempfilename", SQLITE_FCNTL_TEMPFILENAME, "" }, ++ { "has_moved", SQLITE_FCNTL_HAS_MOVED, "" }, ++ { "lock_timeout", SQLITE_FCNTL_LOCK_TIMEOUT, "MILLISEC" }, ++ { "reserve_bytes", SQLITE_FCNTL_RESERVE_BYTES, "[N]" }, ++ }; ++ int filectrl = -1; ++ int iCtrl = -1; ++ sqlite3_int64 iRes = 0; /* Integer result to display if rc2==1 */ ++ int isOk = 0; /* 0: usage 1: %lld 2: no-result */ ++ int n2, i; ++ const char *zCmd = 0; ++ const char *zSchema = 0; ++ ++ open_db(p, 0); ++ zCmd = nArg>=2 ? azArg[1] : "help"; ++ ++ if( zCmd[0]=='-' ++ && (strcmp(zCmd,"--schema")==0 || strcmp(zCmd,"-schema")==0) ++ && nArg>=4 ++ ){ ++ zSchema = azArg[2]; ++ for(i=3; i<nArg; i++) azArg[i-2] = azArg[i]; ++ nArg -= 2; ++ zCmd = azArg[1]; ++ } ++ ++ /* The argument can optionally begin with "-" or "--" */ ++ if( zCmd[0]=='-' && zCmd[1] ){ ++ zCmd++; ++ if( zCmd[0]=='-' && zCmd[1] ) zCmd++; ++ } ++ ++ /* --help lists all file-controls */ ++ if( strcmp(zCmd,"help")==0 ){ ++ utf8_printf(p->out, "Available file-controls:\n"); ++ for(i=0; i<ArraySize(aCtrl); i++){ ++ utf8_printf(p->out, " .filectrl %s %s\n", ++ aCtrl[i].zCtrlName, aCtrl[i].zUsage); ++ } ++ rc = 1; ++ goto meta_command_exit; ++ } ++ ++ /* convert filectrl text option to value. allow any unique prefix ++ ** of the option name, or a numerical value. */ ++ n2 = strlen30(zCmd); ++ for(i=0; i<ArraySize(aCtrl); i++){ ++ if( strncmp(zCmd, aCtrl[i].zCtrlName, n2)==0 ){ ++ if( filectrl<0 ){ ++ filectrl = aCtrl[i].ctrlCode; ++ iCtrl = i; ++ }else{ ++ utf8_printf(stderr, "Error: ambiguous file-control: \"%s\"\n" ++ "Use \".filectrl --help\" for help\n", zCmd); ++ rc = 1; ++ goto meta_command_exit; ++ } ++ } ++ } ++ if( filectrl<0 ){ ++ utf8_printf(stderr,"Error: unknown file-control: %s\n" ++ "Use \".filectrl --help\" for help\n", zCmd); ++ }else{ ++ switch(filectrl){ ++ case SQLITE_FCNTL_SIZE_LIMIT: { ++ if( nArg!=2 && nArg!=3 ) break; ++ iRes = nArg==3 ? integerValue(azArg[2]) : -1; ++ sqlite3_file_control(p->db, zSchema, SQLITE_FCNTL_SIZE_LIMIT, &iRes); ++ isOk = 1; ++ break; ++ } ++ case SQLITE_FCNTL_LOCK_TIMEOUT: ++ case SQLITE_FCNTL_CHUNK_SIZE: { ++ int x; ++ if( nArg!=3 ) break; ++ x = (int)integerValue(azArg[2]); ++ sqlite3_file_control(p->db, zSchema, filectrl, &x); ++ isOk = 2; ++ break; ++ } ++ case SQLITE_FCNTL_PERSIST_WAL: ++ case SQLITE_FCNTL_POWERSAFE_OVERWRITE: { ++ int x; ++ if( nArg!=2 && nArg!=3 ) break; ++ x = nArg==3 ? booleanValue(azArg[2]) : -1; ++ sqlite3_file_control(p->db, zSchema, filectrl, &x); ++ iRes = x; ++ isOk = 1; ++ break; ++ } ++ case SQLITE_FCNTL_HAS_MOVED: { ++ int x; ++ if( nArg!=2 ) break; ++ sqlite3_file_control(p->db, zSchema, filectrl, &x); ++ iRes = x; ++ isOk = 1; ++ break; ++ } ++ case SQLITE_FCNTL_TEMPFILENAME: { ++ char *z = 0; ++ if( nArg!=2 ) break; ++ sqlite3_file_control(p->db, zSchema, filectrl, &z); ++ if( z ){ ++ utf8_printf(p->out, "%s\n", z); ++ sqlite3_free(z); ++ } ++ isOk = 2; ++ break; ++ } ++ case SQLITE_FCNTL_RESERVE_BYTES: { ++ int x; ++ if( nArg>=3 ){ ++ x = atoi(azArg[2]); ++ sqlite3_file_control(p->db, zSchema, filectrl, &x); ++ } ++ x = -1; ++ sqlite3_file_control(p->db, zSchema, filectrl, &x); ++ utf8_printf(p->out,"%d\n", x); ++ isOk = 2; ++ break; ++ } ++ } ++ } ++ if( isOk==0 && iCtrl>=0 ){ ++ utf8_printf(p->out, "Usage: .filectrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage); ++ rc = 1; ++ }else if( isOk==1 ){ ++ char zBuf[100]; ++ sqlite3_snprintf(sizeof(zBuf), zBuf, "%lld", iRes); ++ raw_printf(p->out, "%s\n", zBuf); ++ } ++ }else ++ + if( c=='f' && strncmp(azArg[0], "fullschema", n)==0 ){ + ShellState data; + char *zErrMsg = 0; +@@ -14206,8 +16327,6 @@ + data.cMode = data.mode = MODE_Insert; + data.zDestTable = "sqlite_stat1"; + shell_exec(&data, "SELECT * FROM sqlite_stat1", &zErrMsg); +- data.zDestTable = "sqlite_stat3"; +- shell_exec(&data, "SELECT * FROM sqlite_stat3", &zErrMsg); + data.zDestTable = "sqlite_stat4"; + shell_exec(&data, "SELECT * FROM sqlite_stat4", &zErrMsg); + raw_printf(p->out, "ANALYZE sqlite_master;\n"); +@@ -14235,8 +16354,8 @@ + }else + + if( c=='i' && strncmp(azArg[0], "import", n)==0 ){ +- char *zTable; /* Insert data into this table */ +- char *zFile; /* Name of file to extra content from */ ++ char *zTable = 0; /* Insert data into this table */ ++ char *zFile = 0; /* Name of file to extra content from */ + sqlite3_stmt *pStmt = NULL; /* A statement */ + int nCol; /* Number of columns in the table */ + int nByte; /* Number of bytes in an SQL string */ +@@ -14247,51 +16366,108 @@ + ImportCtx sCtx; /* Reader context */ + char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */ + int (SQLITE_CDECL *xCloser)(FILE*); /* Func to close file */ ++ int eVerbose = 0; /* Larger for more console output */ ++ int nSkip = 0; /* Initial lines to skip */ ++ int useOutputMode = 1; /* Use output mode to determine separators */ + +- if( nArg!=3 ){ +- raw_printf(stderr, "Usage: .import FILE TABLE\n"); +- goto meta_command_exit; +- } +- zFile = azArg[1]; +- zTable = azArg[2]; +- seenInterrupt = 0; + memset(&sCtx, 0, sizeof(sCtx)); +- open_db(p, 0); +- nSep = strlen30(p->colSeparator); +- if( nSep==0 ){ +- raw_printf(stderr, +- "Error: non-null column separator required for import\n"); +- return 1; ++ if( p->mode==MODE_Ascii ){ ++ xRead = ascii_read_one_field; ++ }else{ ++ xRead = csv_read_one_field; + } +- if( nSep>1 ){ +- raw_printf(stderr, "Error: multi-character column separators not allowed" +- " for import\n"); +- return 1; ++ for(i=1; i<nArg; i++){ ++ char *z = azArg[i]; ++ if( z[0]=='-' && z[1]=='-' ) z++; ++ if( z[0]!='-' ){ ++ if( zFile==0 ){ ++ zFile = z; ++ }else if( zTable==0 ){ ++ zTable = z; ++ }else{ ++ utf8_printf(p->out, "ERROR: extra argument: \"%s\". Usage:\n", z); ++ showHelp(p->out, "import"); ++ rc = 1; ++ goto meta_command_exit; ++ } ++ }else if( strcmp(z,"-v")==0 ){ ++ eVerbose++; ++ }else if( strcmp(z,"-skip")==0 && i<nArg-1 ){ ++ nSkip = integerValue(azArg[++i]); ++ }else if( strcmp(z,"-ascii")==0 ){ ++ sCtx.cColSep = SEP_Unit[0]; ++ sCtx.cRowSep = SEP_Record[0]; ++ xRead = ascii_read_one_field; ++ useOutputMode = 0; ++ }else if( strcmp(z,"-csv")==0 ){ ++ sCtx.cColSep = ','; ++ sCtx.cRowSep = '\n'; ++ xRead = csv_read_one_field; ++ useOutputMode = 0; ++ }else{ ++ utf8_printf(p->out, "ERROR: unknown option: \"%s\". Usage:\n", z); ++ showHelp(p->out, "import"); ++ rc = 1; ++ goto meta_command_exit; ++ } + } +- nSep = strlen30(p->rowSeparator); +- if( nSep==0 ){ +- raw_printf(stderr, "Error: non-null row separator required for import\n"); +- return 1; ++ if( zTable==0 ){ ++ utf8_printf(p->out, "ERROR: missing %s argument. Usage:\n", ++ zFile==0 ? "FILE" : "TABLE"); ++ showHelp(p->out, "import"); ++ rc = 1; ++ goto meta_command_exit; + } +- if( nSep==2 && p->mode==MODE_Csv && strcmp(p->rowSeparator, SEP_CrLf)==0 ){ +- /* When importing CSV (only), if the row separator is set to the +- ** default output row separator, change it to the default input +- ** row separator. This avoids having to maintain different input +- ** and output row separators. */ +- sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); ++ seenInterrupt = 0; ++ open_db(p, 0); ++ if( useOutputMode ){ ++ /* If neither the --csv or --ascii options are specified, then set ++ ** the column and row separator characters from the output mode. */ ++ nSep = strlen30(p->colSeparator); ++ if( nSep==0 ){ ++ raw_printf(stderr, ++ "Error: non-null column separator required for import\n"); ++ rc = 1; ++ goto meta_command_exit; ++ } ++ if( nSep>1 ){ ++ raw_printf(stderr, ++ "Error: multi-character column separators not allowed" ++ " for import\n"); ++ rc = 1; ++ goto meta_command_exit; ++ } + nSep = strlen30(p->rowSeparator); +- } +- if( nSep>1 ){ +- raw_printf(stderr, "Error: multi-character row separators not allowed" +- " for import\n"); +- return 1; ++ if( nSep==0 ){ ++ raw_printf(stderr, ++ "Error: non-null row separator required for import\n"); ++ rc = 1; ++ goto meta_command_exit; ++ } ++ if( nSep==2 && p->mode==MODE_Csv && strcmp(p->rowSeparator,SEP_CrLf)==0 ){ ++ /* When importing CSV (only), if the row separator is set to the ++ ** default output row separator, change it to the default input ++ ** row separator. This avoids having to maintain different input ++ ** and output row separators. */ ++ sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); ++ nSep = strlen30(p->rowSeparator); ++ } ++ if( nSep>1 ){ ++ raw_printf(stderr, "Error: multi-character row separators not allowed" ++ " for import\n"); ++ rc = 1; ++ goto meta_command_exit; ++ } ++ sCtx.cColSep = p->colSeparator[0]; ++ sCtx.cRowSep = p->rowSeparator[0]; + } + sCtx.zFile = zFile; + sCtx.nLine = 1; + if( sCtx.zFile[0]=='|' ){ + #ifdef SQLITE_OMIT_POPEN + raw_printf(stderr, "Error: pipes are not supported in this OS\n"); +- return 1; ++ rc = 1; ++ goto meta_command_exit; + #else + sCtx.in = popen(sCtx.zFile+1, "r"); + sCtx.zFile = "<pipe>"; +@@ -14301,17 +16477,26 @@ + sCtx.in = fopen(sCtx.zFile, "rb"); + xCloser = fclose; + } +- if( p->mode==MODE_Ascii ){ +- xRead = ascii_read_one_field; +- }else{ +- xRead = csv_read_one_field; +- } + if( sCtx.in==0 ){ + utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile); +- return 1; ++ rc = 1; ++ goto meta_command_exit; ++ } ++ if( eVerbose>=2 || (eVerbose>=1 && useOutputMode) ){ ++ char zSep[2]; ++ zSep[1] = 0; ++ zSep[0] = sCtx.cColSep; ++ utf8_printf(p->out, "Column separator "); ++ output_c_string(p->out, zSep); ++ utf8_printf(p->out, ", row separator "); ++ zSep[0] = sCtx.cRowSep; ++ output_c_string(p->out, zSep); ++ utf8_printf(p->out, "\n"); ++ } ++ while( (nSkip--)>0 ){ ++ while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){} ++ sCtx.nLine++; + } +- sCtx.cColSep = p->colSeparator[0]; +- sCtx.cRowSep = p->rowSeparator[0]; + zSql = sqlite3_mprintf("SELECT * FROM %s", zTable); + if( zSql==0 ){ + xCloser(sCtx.in); +@@ -14333,9 +16518,13 @@ + sqlite3_free(sCtx.z); + xCloser(sCtx.in); + utf8_printf(stderr,"%s: empty file\n", sCtx.zFile); +- return 1; ++ rc = 1; ++ goto meta_command_exit; + } + zCreate = sqlite3_mprintf("%z\n)", zCreate); ++ if( eVerbose>=1 ){ ++ utf8_printf(p->out, "%s\n", zCreate); ++ } + rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); + sqlite3_free(zCreate); + if( rc ){ +@@ -14343,7 +16532,8 @@ + sqlite3_errmsg(p->db)); + sqlite3_free(sCtx.z); + xCloser(sCtx.in); +- return 1; ++ rc = 1; ++ goto meta_command_exit; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + } +@@ -14352,7 +16542,8 @@ + if (pStmt) sqlite3_finalize(pStmt); + utf8_printf(stderr,"Error: %s\n", sqlite3_errmsg(p->db)); + xCloser(sCtx.in); +- return 1; ++ rc = 1; ++ goto meta_command_exit; + } + nCol = sqlite3_column_count(pStmt); + sqlite3_finalize(pStmt); +@@ -14371,13 +16562,17 @@ + } + zSql[j++] = ')'; + zSql[j] = 0; ++ if( eVerbose>=2 ){ ++ utf8_printf(p->out, "Insert using: %s\n", zSql); ++ } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + if (pStmt) sqlite3_finalize(pStmt); + xCloser(sCtx.in); +- return 1; ++ rc = 1; ++ goto meta_command_exit; + } + needCommit = sqlite3_get_autocommit(p->db); + if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0); +@@ -14420,6 +16615,9 @@ + if( rc!=SQLITE_OK ){ + utf8_printf(stderr, "%s:%d: INSERT failed: %s\n", sCtx.zFile, + startLine, sqlite3_errmsg(p->db)); ++ sCtx.nErr++; ++ }else{ ++ sCtx.nRow++; + } + } + }while( sCtx.cTerm!=EOF ); +@@ -14428,6 +16626,11 @@ + sqlite3_free(sCtx.z); + sqlite3_finalize(pStmt); + if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0); ++ if( eVerbose>0 ){ ++ utf8_printf(p->out, ++ "Added %d rows with %d errors using %d lines of input\n", ++ sCtx.nRow, sCtx.nErr, sCtx.nLine-1); ++ } + }else + + #ifndef SQLITE_UNTESTABLE +@@ -14436,10 +16639,19 @@ + char *zCollist = 0; + sqlite3_stmt *pStmt; + int tnum = 0; ++ int isWO = 0; /* True if making an imposter of a WITHOUT ROWID table */ ++ int lenPK = 0; /* Length of the PRIMARY KEY string for isWO tables */ + int i; + if( !(nArg==3 || (nArg==2 && sqlite3_stricmp(azArg[1],"off")==0)) ){ + utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n" + " .imposter off\n"); ++ /* Also allowed, but not documented: ++ ** ++ ** .imposter TABLE IMPOSTER ++ ** ++ ** where TABLE is a WITHOUT ROWID table. In that case, the ++ ** imposter is another WITHOUT ROWID table with the columns in ++ ** storage order. */ + rc = 1; + goto meta_command_exit; + } +@@ -14448,19 +16660,22 @@ + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 1); + goto meta_command_exit; + } +- zSql = sqlite3_mprintf("SELECT rootpage FROM sqlite_master" +- " WHERE name='%q' AND type='index'", azArg[1]); ++ zSql = sqlite3_mprintf( ++ "SELECT rootpage, 0 FROM sqlite_master" ++ " WHERE name='%q' AND type='index'" ++ "UNION ALL " ++ "SELECT rootpage, 1 FROM sqlite_master" ++ " WHERE name='%q' AND type='table'" ++ " AND sql LIKE '%%without%%rowid%%'", ++ azArg[1], azArg[1] ++ ); + sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + tnum = sqlite3_column_int(pStmt, 0); ++ isWO = sqlite3_column_int(pStmt, 1); + } + sqlite3_finalize(pStmt); +- if( tnum==0 ){ +- utf8_printf(stderr, "no such index: \"%s\"\n", azArg[1]); +- rc = 1; +- goto meta_command_exit; +- } + zSql = sqlite3_mprintf("PRAGMA index_xinfo='%q'", azArg[1]); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); +@@ -14477,6 +16692,9 @@ + zCol = zLabel; + } + } ++ if( isWO && lenPK==0 && sqlite3_column_int(pStmt,5)==0 && zCollist ){ ++ lenPK = (int)strlen(zCollist); ++ } + if( zCollist==0 ){ + zCollist = sqlite3_mprintf("\"%w\"", zCol); + }else{ +@@ -14484,9 +16702,16 @@ + } + } + sqlite3_finalize(pStmt); ++ if( i==0 || tnum==0 ){ ++ utf8_printf(stderr, "no such index: \"%s\"\n", azArg[1]); ++ rc = 1; ++ sqlite3_free(zCollist); ++ goto meta_command_exit; ++ } ++ if( lenPK==0 ) lenPK = 100000; + zSql = sqlite3_mprintf( +- "CREATE TABLE \"%w\"(%s,PRIMARY KEY(%s))WITHOUT ROWID", +- azArg[2], zCollist, zCollist); ++ "CREATE TABLE \"%w\"(%s,PRIMARY KEY(%.*s))WITHOUT ROWID", ++ azArg[2], zCollist, lenPK, zCollist); + sqlite3_free(zCollist); + rc = sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 1, tnum); + if( rc==SQLITE_OK ){ +@@ -14497,7 +16722,8 @@ + }else{ + utf8_printf(stdout, "%s;\n", zSql); + raw_printf(stdout, +- "WARNING: writing to an imposter table will corrupt the index!\n" ++ "WARNING: writing to an imposter table will corrupt the \"%s\" %s!\n", ++ azArg[1], isWO ? "table" : "index" + ); + } + }else{ +@@ -14683,6 +16909,34 @@ + } + }else + ++#ifdef SQLITE_DEBUG ++ if( c=='o' && strcmp(azArg[0],"oom")==0 ){ ++ int i; ++ for(i=1; i<nArg; i++){ ++ const char *z = azArg[i]; ++ if( z[0]=='-' && z[1]=='-' ) z++; ++ if( strcmp(z,"-repeat")==0 ){ ++ if( i==nArg-1 ){ ++ raw_printf(p->out, "missing argument on \"%s\"\n", azArg[i]); ++ rc = 1; ++ }else{ ++ oomRepeat = (int)integerValue(azArg[++i]); ++ } ++ }else if( IsDigit(z[0]) ){ ++ oomCounter = (int)integerValue(azArg[i]); ++ }else{ ++ raw_printf(p->out, "unknown argument: \"%s\"\n", azArg[i]); ++ raw_printf(p->out, "Usage: .oom [--repeat N] [M]\n"); ++ rc = 1; ++ } ++ } ++ if( rc==0 ){ ++ raw_printf(p->out, "oomCounter = %d\n", oomCounter); ++ raw_printf(p->out, "oomRepeat = %d\n", oomRepeat); ++ } ++ }else ++#endif /* SQLITE_DEBUG */ ++ + if( c=='o' && strncmp(azArg[0], "open", n)==0 && n>=2 ){ + char *zNewFilename; /* Name of the database file to open */ + int iName = 1; /* Index in azArg[] of the filename */ +@@ -14695,6 +16949,7 @@ + sqlite3_free(p->zFreeOnClose); + p->zFreeOnClose = 0; + p->openMode = SHELL_OPEN_UNSPEC; ++ p->openFlags = 0; + p->szMax = 0; + /* Check for command-line arguments */ + for(iName=1; iName<nArg && azArg[iName][0]=='-'; iName++){ +@@ -14709,6 +16964,8 @@ + p->openMode = SHELL_OPEN_APPENDVFS; + }else if( optionMatch(z, "readonly") ){ + p->openMode = SHELL_OPEN_READONLY; ++ }else if( optionMatch(z, "nofollow") ){ ++ p->openFlags |= SQLITE_OPEN_NOFOLLOW; + #ifdef SQLITE_ENABLE_DESERIALIZE + }else if( optionMatch(z, "deserialize") ){ + p->openMode = SHELL_OPEN_DESERIALIZE; +@@ -14747,42 +17004,66 @@ + && (strncmp(azArg[0], "output", n)==0||strncmp(azArg[0], "once", n)==0)) + || (c=='e' && n==5 && strcmp(azArg[0],"excel")==0) + ){ +- const char *zFile = nArg>=2 ? azArg[1] : "stdout"; ++ const char *zFile = 0; + int bTxtMode = 0; +- if( azArg[0][0]=='e' ){ +- /* Transform the ".excel" command into ".once -x" */ +- nArg = 2; +- azArg[0] = "once"; +- zFile = azArg[1] = "-x"; +- n = 4; +- } +- if( nArg>2 ){ +- utf8_printf(stderr, "Usage: .%s [-e|-x|FILE]\n", azArg[0]); +- rc = 1; +- goto meta_command_exit; ++ int i; ++ int eMode = 0; ++ int bBOM = 0; ++ int bOnce = 0; /* 0: .output, 1: .once, 2: .excel */ ++ ++ if( c=='e' ){ ++ eMode = 'x'; ++ bOnce = 2; ++ }else if( strncmp(azArg[0],"once",n)==0 ){ ++ bOnce = 1; + } +- if( n>1 && strncmp(azArg[0], "once", n)==0 ){ +- if( nArg<2 ){ +- raw_printf(stderr, "Usage: .once (-e|-x|FILE)\n"); ++ for(i=1; i<nArg; i++){ ++ char *z = azArg[i]; ++ if( z[0]=='-' ){ ++ if( z[1]=='-' ) z++; ++ if( strcmp(z,"-bom")==0 ){ ++ bBOM = 1; ++ }else if( c!='e' && strcmp(z,"-x")==0 ){ ++ eMode = 'x'; /* spreadsheet */ ++ }else if( c!='e' && strcmp(z,"-e")==0 ){ ++ eMode = 'e'; /* text editor */ ++ }else{ ++ utf8_printf(p->out, "ERROR: unknown option: \"%s\". Usage:\n", ++ azArg[i]); ++ showHelp(p->out, azArg[0]); ++ rc = 1; ++ goto meta_command_exit; ++ } ++ }else if( zFile==0 ){ ++ zFile = z; ++ }else{ ++ utf8_printf(p->out,"ERROR: extra parameter: \"%s\". Usage:\n", ++ azArg[i]); ++ showHelp(p->out, azArg[0]); + rc = 1; + goto meta_command_exit; + } ++ } ++ if( zFile==0 ) zFile = "stdout"; ++ if( bOnce ){ + p->outCount = 2; + }else{ + p->outCount = 0; + } + output_reset(p); +- if( zFile[0]=='-' && zFile[1]=='-' ) zFile++; + #ifndef SQLITE_NOHAVE_SYSTEM +- if( strcmp(zFile, "-e")==0 || strcmp(zFile, "-x")==0 ){ ++ if( eMode=='e' || eMode=='x' ){ + p->doXdgOpen = 1; + outputModePush(p); +- if( zFile[1]=='x' ){ ++ if( eMode=='x' ){ ++ /* spreadsheet mode. Output as CSV. */ + newTempFile(p, "csv"); ++ ShellClearFlag(p, SHFLG_Echo); + p->mode = MODE_Csv; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf); + }else{ ++ /* text editor mode */ + newTempFile(p, "txt"); + bTxtMode = 1; + } +@@ -14801,6 +17082,7 @@ + p->out = stdout; + rc = 1; + }else{ ++ if( bBOM ) fprintf(p->out,"\357\273\277"); + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); + } + #endif +@@ -14813,6 +17095,7 @@ + p->out = stdout; + rc = 1; + } else { ++ if( bBOM ) fprintf(p->out,"\357\273\277"); + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); + } + } +@@ -14826,12 +17109,8 @@ + ** Clear all bind parameters by dropping the TEMP table that holds them. + */ + if( nArg==2 && strcmp(azArg[1],"clear")==0 ){ +- int wrSchema = 0; +- sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, -1, &wrSchema); +- sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, 1, 0); + sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp.sqlite_parameters;", + 0, 0, 0); +- sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, wrSchema, 0); + }else + + /* .parameter list +@@ -15144,7 +17423,7 @@ + zDiv = " UNION ALL "; + appendText(&sSelect, "SELECT shell_add_schema(sql,", 0); + if( sqlite3_stricmp(zDb, "main")!=0 ){ +- appendText(&sSelect, zDb, '"'); ++ appendText(&sSelect, zDb, '\''); + }else{ + appendText(&sSelect, "NULL", 0); + } +@@ -15153,15 +17432,16 @@ + appendText(&sSelect, " AS snum, ", 0); + appendText(&sSelect, zDb, '\''); + appendText(&sSelect, " AS sname FROM ", 0); +- appendText(&sSelect, zDb, '"'); ++ appendText(&sSelect, zDb, quoteChar(zDb)); + appendText(&sSelect, ".sqlite_master", 0); + } + sqlite3_finalize(pStmt); +-#ifdef SQLITE_INTROSPECTION_PRAGMAS ++#ifndef SQLITE_OMIT_INTROSPECTION_PRAGMAS + if( zName ){ + appendText(&sSelect, + " UNION ALL SELECT shell_module_schema(name)," +- " 'table', name, name, name, 9e+99, 'main' FROM pragma_module_list", 0); ++ " 'table', name, name, name, 9e+99, 'main' FROM pragma_module_list", ++ 0); + } + #endif + appendText(&sSelect, ") WHERE ", 0); +@@ -15260,7 +17540,8 @@ + if( pSession->p==0 ) goto session_not_open; + out = fopen(azCmd[1], "wb"); + if( out==0 ){ +- utf8_printf(stderr, "ERROR: cannot open \"%s\" for writing\n", azCmd[1]); ++ utf8_printf(stderr, "ERROR: cannot open \"%s\" for writing\n", ++ azCmd[1]); + }else{ + int szChng; + void *pChng; +@@ -15581,8 +17862,7 @@ + { + utf8_printf(stderr, "Unknown option \"%s\" on \"%s\"\n", + azArg[i], azArg[0]); +- raw_printf(stderr, "Should be one of: --schema" +- " --sha3-224 --sha3-256 --sha3-384 --sha3-512\n"); ++ showHelp(p->out, azArg[0]); + rc = 1; + goto meta_command_exit; + } +@@ -15628,8 +17908,7 @@ + }else if( strcmp(zTab, "sqlite_stat1")==0 ){ + appendText(&sQuery,"SELECT tbl,idx,stat FROM sqlite_stat1" + " ORDER BY tbl,idx;", 0); +- }else if( strcmp(zTab, "sqlite_stat3")==0 +- || strcmp(zTab, "sqlite_stat4")==0 ){ ++ }else if( strcmp(zTab, "sqlite_stat4")==0 ){ + appendText(&sQuery, "SELECT * FROM ", 0); + appendText(&sQuery, zTab, 0); + appendText(&sQuery, " ORDER BY tbl, idx, rowid;\n", 0); +@@ -15861,25 +18140,25 @@ + int ctrlCode; /* Integer code for that option */ + const char *zUsage; /* Usage notes */ + } aCtrl[] = { +- { "always", SQLITE_TESTCTRL_ALWAYS, "BOOLEAN" }, +- { "assert", SQLITE_TESTCTRL_ASSERT, "BOOLEAN" }, +- /*{ "benign_malloc_hooks",SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS, "" },*/ +- /*{ "bitvec_test", SQLITE_TESTCTRL_BITVEC_TEST, "" },*/ +- { "byteorder", SQLITE_TESTCTRL_BYTEORDER, "" }, +- /*{ "fault_install", SQLITE_TESTCTRL_FAULT_INSTALL, "" }, */ +- { "imposter", SQLITE_TESTCTRL_IMPOSTER, "SCHEMA ON/OFF ROOTPAGE"}, +- { "internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS, "BOOLEAN" }, +- { "localtime_fault", SQLITE_TESTCTRL_LOCALTIME_FAULT,"BOOLEAN" }, +- { "never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT, "BOOLEAN" }, +- { "optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS, "DISABLE-MASK" }, ++ { "always", SQLITE_TESTCTRL_ALWAYS, "BOOLEAN" }, ++ { "assert", SQLITE_TESTCTRL_ASSERT, "BOOLEAN" }, ++ /*{ "benign_malloc_hooks",SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS, "" },*/ ++ /*{ "bitvec_test", SQLITE_TESTCTRL_BITVEC_TEST, "" },*/ ++ { "byteorder", SQLITE_TESTCTRL_BYTEORDER, "" }, ++ { "extra_schema_checks",SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS,"BOOLEAN" }, ++ /*{ "fault_install", SQLITE_TESTCTRL_FAULT_INSTALL, "" },*/ ++ { "imposter", SQLITE_TESTCTRL_IMPOSTER, "SCHEMA ON/OFF ROOTPAGE"}, ++ { "internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS, "" }, ++ { "localtime_fault", SQLITE_TESTCTRL_LOCALTIME_FAULT,"BOOLEAN" }, ++ { "never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT, "BOOLEAN" }, ++ { "optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS, "DISABLE-MASK" }, + #ifdef YYCOVERAGE +- { "parser_coverage", SQLITE_TESTCTRL_PARSER_COVERAGE, "" }, ++ { "parser_coverage", SQLITE_TESTCTRL_PARSER_COVERAGE, "" }, + #endif +- { "pending_byte", SQLITE_TESTCTRL_PENDING_BYTE, "OFFSET " }, +- { "prng_reset", SQLITE_TESTCTRL_PRNG_RESET, "" }, +- { "prng_restore", SQLITE_TESTCTRL_PRNG_RESTORE, "" }, +- { "prng_save", SQLITE_TESTCTRL_PRNG_SAVE, "" }, +- { "reserve", SQLITE_TESTCTRL_RESERVE, "BYTES-OF-RESERVE" }, ++ { "pending_byte", SQLITE_TESTCTRL_PENDING_BYTE, "OFFSET " }, ++ { "prng_restore", SQLITE_TESTCTRL_PRNG_RESTORE, "" }, ++ { "prng_save", SQLITE_TESTCTRL_PRNG_SAVE, "" }, ++ { "prng_seed", SQLITE_TESTCTRL_PRNG_SEED, "SEED ?db?" }, + }; + int testctrl = -1; + int iCtrl = -1; +@@ -15932,7 +18211,6 @@ + + /* sqlite3_test_control(int, db, int) */ + case SQLITE_TESTCTRL_OPTIMIZATIONS: +- case SQLITE_TESTCTRL_RESERVE: + if( nArg==3 ){ + int opt = (int)strtol(azArg[2], 0, 0); + rc2 = sqlite3_test_control(testctrl, p->db, opt); +@@ -15960,10 +18238,30 @@ + } + break; + ++ /* sqlite3_test_control(int, int, sqlite3*) */ ++ case SQLITE_TESTCTRL_PRNG_SEED: ++ if( nArg==3 || nArg==4 ){ ++ int ii = (int)integerValue(azArg[2]); ++ sqlite3 *db; ++ if( ii==0 && strcmp(azArg[2],"random")==0 ){ ++ sqlite3_randomness(sizeof(ii),&ii); ++ printf("-- random seed: %d\n", ii); ++ } ++ if( nArg==3 ){ ++ db = 0; ++ }else{ ++ db = p->db; ++ /* Make sure the schema has been loaded */ ++ sqlite3_table_column_metadata(db, 0, "x", 0, 0, 0, 0, 0, 0); ++ } ++ rc2 = sqlite3_test_control(testctrl, ii, db); ++ isOk = 3; ++ } ++ break; ++ + /* sqlite3_test_control(int, int) */ + case SQLITE_TESTCTRL_ASSERT: + case SQLITE_TESTCTRL_ALWAYS: +- case SQLITE_TESTCTRL_INTERNAL_FUNCTIONS: + if( nArg==3 ){ + int opt = booleanValue(azArg[2]); + rc2 = sqlite3_test_control(testctrl, opt); +@@ -15981,6 +18279,12 @@ + } + break; + ++ /* sqlite3_test_control(sqlite3*) */ ++ case SQLITE_TESTCTRL_INTERNAL_FUNCTIONS: ++ rc2 = sqlite3_test_control(testctrl, p->db); ++ isOk = 3; ++ break; ++ + case SQLITE_TESTCTRL_IMPOSTER: + if( nArg==5 ){ + rc2 = sqlite3_test_control(testctrl, p->db, +@@ -16001,7 +18305,7 @@ + } + } + if( isOk==0 && iCtrl>=0 ){ +- utf8_printf(p->out, "Usage: .testctrl %s %s\n", zCmd, aCtrl[iCtrl].zUsage); ++ utf8_printf(p->out, "Usage: .testctrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage); + rc = 1; + }else if( isOk==1 ){ + raw_printf(p->out, "%d\n", rc2); +@@ -16079,6 +18383,31 @@ + }else + #endif /* !defined(SQLITE_OMIT_TRACE) */ + ++#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_VIRTUALTABLE) ++ if( c=='u' && strncmp(azArg[0], "unmodule", n)==0 ){ ++ int ii; ++ int lenOpt; ++ char *zOpt; ++ if( nArg<2 ){ ++ raw_printf(stderr, "Usage: .unmodule [--allexcept] NAME ...\n"); ++ rc = 1; ++ goto meta_command_exit; ++ } ++ open_db(p, 0); ++ zOpt = azArg[1]; ++ if( zOpt[0]=='-' && zOpt[1]=='-' && zOpt[2]!=0 ) zOpt++; ++ lenOpt = (int)strlen(zOpt); ++ if( lenOpt>=3 && strncmp(zOpt, "-allexcept",lenOpt)==0 ){ ++ assert( azArg[nArg]==0 ); ++ sqlite3_drop_modules(p->db, nArg>2 ? (const char**)(azArg+2) : 0); ++ }else{ ++ for(ii=1; ii<nArg; ii++){ ++ sqlite3_create_module(p->db, azArg[ii], 0, 0); ++ } ++ } ++ }else ++#endif ++ + #if SQLITE_USER_AUTHENTICATION + if( c=='u' && strncmp(azArg[0], "user", n)==0 ){ + if( nArg<2 ){ +@@ -16093,7 +18422,8 @@ + rc = 1; + goto meta_command_exit; + } +- rc = sqlite3_user_authenticate(p->db, azArg[2], azArg[3], strlen30(azArg[3])); ++ rc = sqlite3_user_authenticate(p->db, azArg[2], azArg[3], ++ strlen30(azArg[3])); + if( rc ){ + utf8_printf(stderr, "Authentication failed for user %s\n", azArg[2]); + rc = 1; +@@ -16585,6 +18915,7 @@ + " -multiplex enable the multiplexor VFS\n" + #endif + " -newline SEP set output row separator. Default: '\\n'\n" ++ " -nofollow refuse to open symbolic links to database files\n" + " -nullvalue TEXT set text string for NULL values. Default ''\n" + " -pagecache SIZE N use N slots of SZ bytes each for page cache memory\n" + " -quote set output mode to 'quote'\n" +@@ -16651,14 +18982,18 @@ + */ + #ifdef _WIN32 + static void printBold(const char *zText){ ++#if !SQLITE_OS_WINRT + HANDLE out = GetStdHandle(STD_OUTPUT_HANDLE); + CONSOLE_SCREEN_BUFFER_INFO defaultScreenInfo; + GetConsoleScreenBufferInfo(out, &defaultScreenInfo); + SetConsoleTextAttribute(out, + FOREGROUND_RED|FOREGROUND_INTENSITY + ); ++#endif + printf("%s", zText); ++#if !SQLITE_OS_WINRT + SetConsoleTextAttribute(out, defaultScreenInfo.wAttributes); ++#endif + } + #else + static void printBold(const char *zText){ +@@ -16713,6 +19048,10 @@ + stdin_is_interactive = isatty(0); + stdout_is_console = isatty(1); + ++#ifdef SQLITE_DEBUG ++ registerOomSimulator(); ++#endif ++ + #if !defined(_WIN32_WCE) + if( getenv("SQLITE_DEBUG_BREAK") ){ + if( isatty(0) && isatty(2) ){ +@@ -16722,7 +19061,11 @@ + fgetc(stdin); + }else{ + #if defined(_WIN32) || defined(WIN32) ++#if SQLITE_OS_WINRT ++ __debugbreak(); ++#else + DebugBreak(); ++#endif + #elif defined(SIGTRAP) + raise(SIGTRAP); + #endif +@@ -16895,6 +19238,8 @@ + #endif + }else if( strcmp(z,"-readonly")==0 ){ + data.openMode = SHELL_OPEN_READONLY; ++ }else if( strcmp(z,"-nofollow")==0 ){ ++ data.openFlags = SQLITE_OPEN_NOFOLLOW; + #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) + }else if( strncmp(z, "-A",2)==0 ){ + /* All remaining command-line arguments are passed to the ".archive" +@@ -16998,6 +19343,8 @@ + #endif + }else if( strcmp(z,"-readonly")==0 ){ + data.openMode = SHELL_OPEN_READONLY; ++ }else if( strcmp(z,"-nofollow")==0 ){ ++ data.openFlags |= SQLITE_OPEN_NOFOLLOW; + }else if( strcmp(z,"-ascii")==0 ){ + data.mode = MODE_Ascii; + sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, +--- contrib/sqlite3/sqlite3.c.orig ++++ contrib/sqlite3/sqlite3.c +@@ -1,6 +1,6 @@ + /****************************************************************************** + ** This file is an amalgamation of many separate C source files from SQLite +-** version 3.28.0. By combining all the individual C code files into this ++** version 3.32.2. By combining all the individual C code files into this + ** single large file, the entire code can be compiled as a single translation + ** unit. This allows many compilers to do optimizations that would not be + ** possible if the files were compiled separately. Performance improvements +@@ -39,7 +39,7 @@ + ** SQLite was built with. + */ + +-#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS ++#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS /* IMP: R-16824-07538 */ + + /* + ** Include the configuration header output by 'configure' if we're using the +@@ -218,6 +218,9 @@ + #if SQLITE_ENABLE_BATCH_ATOMIC_WRITE + "ENABLE_BATCH_ATOMIC_WRITE", + #endif ++#if SQLITE_ENABLE_BYTECODE_VTAB ++ "ENABLE_BYTECODE_VTAB", ++#endif + #if SQLITE_ENABLE_CEROD + "ENABLE_CEROD=" CTIMEOPT_VAL(SQLITE_ENABLE_CEROD), + #endif +@@ -331,8 +334,6 @@ + #endif + #if defined(SQLITE_ENABLE_STAT4) + "ENABLE_STAT4", +-#elif defined(SQLITE_ENABLE_STAT3) +- "ENABLE_STAT3", + #endif + #if SQLITE_ENABLE_STMTVTAB + "ENABLE_STMTVTAB", +@@ -382,9 +383,6 @@ + #if SQLITE_FTS5_NO_WITHOUT_ROWID + "FTS5_NO_WITHOUT_ROWID", + #endif +-#if SQLITE_HAS_CODEC +- "HAS_CODEC", +-#endif + #if HAVE_ISNAN || SQLITE_HAVE_ISNAN + "HAVE_ISNAN", + #endif +@@ -541,9 +539,6 @@ + #if SQLITE_OMIT_BLOB_LITERAL + "OMIT_BLOB_LITERAL", + #endif +-#if SQLITE_OMIT_BTREECOUNT +- "OMIT_BTREECOUNT", +-#endif + #if SQLITE_OMIT_CAST + "OMIT_CAST", + #endif +@@ -888,6 +883,11 @@ + #pragma warning(disable : 4706) + #endif /* defined(_MSC_VER) */ + ++#if defined(_MSC_VER) && !defined(_WIN64) ++#undef SQLITE_4_BYTE_ALIGNED_MALLOC ++#define SQLITE_4_BYTE_ALIGNED_MALLOC ++#endif /* defined(_MSC_VER) && !defined(_WIN64) */ ++ + #endif /* SQLITE_MSVC_H */ + + /************** End of msvc.h ************************************************/ +@@ -1162,9 +1162,9 @@ + ** [sqlite3_libversion_number()], [sqlite3_sourceid()], + ** [sqlite_version()] and [sqlite_source_id()]. + */ +-#define SQLITE_VERSION "3.28.0" +-#define SQLITE_VERSION_NUMBER 3028000 +-#define SQLITE_SOURCE_ID "2019-04-16 19:49:53 884b4b7e502b4e991677b53971277adfaf0a04a284f8e483e2553d0f83156b50" ++#define SQLITE_VERSION "3.32.2" ++#define SQLITE_VERSION_NUMBER 3032002 ++#define SQLITE_SOURCE_ID "2020-06-04 12:58:43 ec02243ea6ce33b090870ae55ab8aa2534b54d216d45c4aa2fdbb00e86861e8c" + + /* + ** CAPI3REF: Run-Time Library Version Numbers +@@ -1338,26 +1338,22 @@ + ** the [sqlite3] object is successfully destroyed and all associated + ** resources are deallocated. + ** +-** ^If the database connection is associated with unfinalized prepared +-** statements or unfinished sqlite3_backup objects then sqlite3_close() +-** will leave the database connection open and return [SQLITE_BUSY]. +-** ^If sqlite3_close_v2() is called with unfinalized prepared statements +-** and/or unfinished sqlite3_backups, then the database connection becomes +-** an unusable "zombie" which will automatically be deallocated when the +-** last prepared statement is finalized or the last sqlite3_backup is +-** finished. The sqlite3_close_v2() interface is intended for use with +-** host languages that are garbage collected, and where the order in which +-** destructors are called is arbitrary. +-** +-** Applications should [sqlite3_finalize | finalize] all [prepared statements], +-** [sqlite3_blob_close | close] all [BLOB handles], and ++** Ideally, applications should [sqlite3_finalize | finalize] all ++** [prepared statements], [sqlite3_blob_close | close] all [BLOB handles], and + ** [sqlite3_backup_finish | finish] all [sqlite3_backup] objects associated +-** with the [sqlite3] object prior to attempting to close the object. ^If +-** sqlite3_close_v2() is called on a [database connection] that still has +-** outstanding [prepared statements], [BLOB handles], and/or +-** [sqlite3_backup] objects then it returns [SQLITE_OK] and the deallocation +-** of resources is deferred until all [prepared statements], [BLOB handles], +-** and [sqlite3_backup] objects are also destroyed. ++** with the [sqlite3] object prior to attempting to close the object. ++** ^If the database connection is associated with unfinalized prepared ++** statements, BLOB handlers, and/or unfinished sqlite3_backup objects then ++** sqlite3_close() will leave the database connection open and return ++** [SQLITE_BUSY]. ^If sqlite3_close_v2() is called with unfinalized prepared ++** statements, unclosed BLOB handlers, and/or unfinished sqlite3_backups, ++** it returns [SQLITE_OK] regardless, but instead of deallocating the database ++** connection immediately, it marks the database connection as an unusable ++** "zombie" and makes arrangements to automatically deallocate the database ++** connection after all prepared statements are finalized, all BLOB handles ++** are closed, and all backups have finished. The sqlite3_close_v2() interface ++** is intended for use with host languages that are garbage collected, and ++** where the order in which destructors are called is arbitrary. + ** + ** ^If an [sqlite3] object is destroyed while a transaction is open, + ** the transaction is automatically rolled back. +@@ -1546,17 +1542,21 @@ + #define SQLITE_IOERR_BEGIN_ATOMIC (SQLITE_IOERR | (29<<8)) + #define SQLITE_IOERR_COMMIT_ATOMIC (SQLITE_IOERR | (30<<8)) + #define SQLITE_IOERR_ROLLBACK_ATOMIC (SQLITE_IOERR | (31<<8)) ++#define SQLITE_IOERR_DATA (SQLITE_IOERR | (32<<8)) + #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) + #define SQLITE_LOCKED_VTAB (SQLITE_LOCKED | (2<<8)) + #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) + #define SQLITE_BUSY_SNAPSHOT (SQLITE_BUSY | (2<<8)) ++#define SQLITE_BUSY_TIMEOUT (SQLITE_BUSY | (3<<8)) + #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) + #define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) + #define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) + #define SQLITE_CANTOPEN_CONVPATH (SQLITE_CANTOPEN | (4<<8)) + #define SQLITE_CANTOPEN_DIRTYWAL (SQLITE_CANTOPEN | (5<<8)) /* Not Used */ ++#define SQLITE_CANTOPEN_SYMLINK (SQLITE_CANTOPEN | (6<<8)) + #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) + #define SQLITE_CORRUPT_SEQUENCE (SQLITE_CORRUPT | (2<<8)) ++#define SQLITE_CORRUPT_INDEX (SQLITE_CORRUPT | (3<<8)) + #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) + #define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) + #define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) +@@ -1574,11 +1574,13 @@ + #define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) + #define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) + #define SQLITE_CONSTRAINT_ROWID (SQLITE_CONSTRAINT |(10<<8)) ++#define SQLITE_CONSTRAINT_PINNED (SQLITE_CONSTRAINT |(11<<8)) + #define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) + #define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) + #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) + #define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) + #define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) ++#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) + + /* + ** CAPI3REF: Flags For File Open Operations +@@ -1607,6 +1609,7 @@ + #define SQLITE_OPEN_SHAREDCACHE 0x00020000 /* Ok for sqlite3_open_v2() */ + #define SQLITE_OPEN_PRIVATECACHE 0x00040000 /* Ok for sqlite3_open_v2() */ + #define SQLITE_OPEN_WAL 0x00080000 /* VFS only */ ++#define SQLITE_OPEN_NOFOLLOW 0x01000000 /* Ok for sqlite3_open_v2() */ + + /* Reserved: 0x00F00000 */ + +@@ -2018,16 +2021,16 @@ + ** ^The [SQLITE_FCNTL_BUSYHANDLER] + ** file-control may be invoked by SQLite on the database file handle + ** shortly after it is opened in order to provide a custom VFS with access +-** to the connections busy-handler callback. The argument is of type (void **) ++** to the connection's busy-handler callback. The argument is of type (void**) + ** - an array of two (void *) values. The first (void *) actually points +-** to a function of type (int (*)(void *)). In order to invoke the connections ++** to a function of type (int (*)(void *)). In order to invoke the connection's + ** busy-handler, this function should be invoked with the second (void *) in + ** the array as the only argument. If it returns non-zero, then the operation + ** should be retried. If it returns zero, the custom VFS should abandon the + ** current operation. + ** + ** <li>[[SQLITE_FCNTL_TEMPFILENAME]] +-** ^Application can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control ++** ^Applications can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control + ** to have SQLite generate a + ** temporary filename using the same algorithm that is followed to generate + ** temporary filenames for TEMP tables and other internal uses. The +@@ -2122,10 +2125,12 @@ + ** a prior successful call to [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE]. + ** + ** <li>[[SQLITE_FCNTL_LOCK_TIMEOUT]] +-** The [SQLITE_FCNTL_LOCK_TIMEOUT] opcode causes attempts to obtain +-** a file lock using the xLock or xShmLock methods of the VFS to wait +-** for up to M milliseconds before failing, where M is the single +-** unsigned integer parameter. ++** The [SQLITE_FCNTL_LOCK_TIMEOUT] opcode is used to configure a VFS ++** to block for up to M milliseconds before failing when attempting to ++** obtain a file lock using the xLock or xShmLock methods of the VFS. ++** The parameter is a pointer to a 32-bit signed integer that contains ++** the value that M is to be set to. Before returning, the 32-bit signed ++** integer is overwritten with the previous value of M. + ** + ** <li>[[SQLITE_FCNTL_DATA_VERSION]] + ** The [SQLITE_FCNTL_DATA_VERSION] opcode is used to detect changes to +@@ -2140,12 +2145,23 @@ + ** not provide a mechanism to detect changes to MAIN only. Also, the + ** [sqlite3_total_changes()] interface responds to internal changes only and + ** omits changes made by other database connections. The +-** [PRAGMA data_version] command provide a mechanism to detect changes to ++** [PRAGMA data_version] command provides a mechanism to detect changes to + ** a single attached database that occur due to other database connections, + ** but omits changes implemented by the database connection on which it is + ** called. This file control is the only mechanism to detect changes that + ** happen either internally or externally and that are associated with + ** a particular attached database. ++** ++** <li>[[SQLITE_FCNTL_CKPT_START]] ++** The [SQLITE_FCNTL_CKPT_START] opcode is invoked from within a checkpoint ++** in wal mode before the client starts to copy pages from the wal ++** file to the database file. ++** ++** <li>[[SQLITE_FCNTL_CKPT_DONE]] ++** The [SQLITE_FCNTL_CKPT_DONE] opcode is invoked from within a checkpoint ++** in wal mode after the client has finished copying pages from the wal ++** file to the database file, but before the *-shm file is updated to ++** record the fact that the pages have been checkpointed. + ** </ul> + */ + #define SQLITE_FCNTL_LOCKSTATE 1 +@@ -2183,6 +2199,9 @@ + #define SQLITE_FCNTL_LOCK_TIMEOUT 34 + #define SQLITE_FCNTL_DATA_VERSION 35 + #define SQLITE_FCNTL_SIZE_LIMIT 36 ++#define SQLITE_FCNTL_CKPT_DONE 37 ++#define SQLITE_FCNTL_RESERVE_BYTES 38 ++#define SQLITE_FCNTL_CKPT_START 39 + + /* deprecated names */ + #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE +@@ -2228,10 +2247,10 @@ + ** to 3 with SQLite [version 3.7.6] on [dateof:3.7.6]. Additional fields + ** may be appended to the sqlite3_vfs object and the iVersion value + ** may increase again in future versions of SQLite. +-** Note that the structure +-** of the sqlite3_vfs object changes in the transition from ++** Note that due to an oversight, the structure ++** of the sqlite3_vfs object changed in the transition from + ** SQLite [version 3.5.9] to [version 3.6.0] on [dateof:3.6.0] +-** and yet the iVersion field was not modified. ++** and yet the iVersion field was not increased. + ** + ** The szOsFile field is the size of the subclassed [sqlite3_file] + ** structure used by this VFS. mxPathname is the maximum length of +@@ -2322,7 +2341,7 @@ + ** for exclusive access. + ** + ** ^At least szOsFile bytes of memory are allocated by SQLite +-** to hold the [sqlite3_file] structure passed as the third ++** to hold the [sqlite3_file] structure passed as the third + ** argument to xOpen. The xOpen method does not have to + ** allocate the structure; it should just fill it in. Note that + ** the xOpen method must set the sqlite3_file.pMethods to either +@@ -2335,8 +2354,14 @@ + ** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] + ** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to + ** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] +-** to test whether a file is at least readable. The file can be a +-** directory. ++** to test whether a file is at least readable. The SQLITE_ACCESS_READ ++** flag is never actually used and is not implemented in the built-in ++** VFSes of SQLite. The file is named by the second argument and can be a ++** directory. The xAccess method returns [SQLITE_OK] on success or some ++** non-zero error code if there is an I/O error or if the name of ++** the file given in the second argument is illegal. If SQLITE_OK ++** is returned, then non-zero or zero is written into *pResOut to indicate ++** whether or not the file is accessible. + ** + ** ^SQLite will always allocate at least mxPathname+1 bytes for the + ** output buffer xFullPathname. The exact size of the output buffer +@@ -2653,7 +2678,7 @@ + ** that causes the corresponding memory allocation to fail. + ** + ** The xInit method initializes the memory allocator. For example, +-** it might allocate any require mutexes or initialize internal data ++** it might allocate any required mutexes or initialize internal data + ** structures. The xShutdown method is invoked (indirectly) by + ** [sqlite3_shutdown()] and should deallocate any resources acquired + ** by xInit. The pAppData pointer is used as the only parameter to +@@ -2775,6 +2800,7 @@ + ** memory allocation statistics. ^(When memory allocation statistics are + ** disabled, the following SQLite interfaces become non-operational: + ** <ul> ++** <li> [sqlite3_hard_heap_limit64()] + ** <li> [sqlite3_memory_used()] + ** <li> [sqlite3_memory_highwater()] + ** <li> [sqlite3_soft_heap_limit64()] +@@ -2793,7 +2819,7 @@ + ** <dd> ^The SQLITE_CONFIG_PAGECACHE option specifies a memory pool + ** that SQLite can use for the database page cache with the default page + ** cache implementation. +-** This configuration option is a no-op if an application-define page ++** This configuration option is a no-op if an application-defined page + ** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2]. + ** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to + ** 8-byte aligned memory (pMem), the size of each page cache line (sz), +@@ -3126,6 +3152,17 @@ + ** following this call. The second parameter may be a NULL pointer, in + ** which case the trigger setting is not reported back. </dd> + ** ++** [[SQLITE_DBCONFIG_ENABLE_VIEW]] ++** <dt>SQLITE_DBCONFIG_ENABLE_VIEW</dt> ++** <dd> ^This option is used to enable or disable [CREATE VIEW | views]. ++** There should be two additional arguments. ++** The first argument is an integer which is 0 to disable views, ++** positive to enable views or negative to leave the setting unchanged. ++** The second parameter is a pointer to an integer into which ++** is written 0 or 1 to indicate whether views are disabled or enabled ++** following this call. The second parameter may be a NULL pointer, in ++** which case the view setting is not reported back. </dd> ++** + ** [[SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER]] + ** <dt>SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER</dt> + ** <dd> ^This option is used to enable or disable the +@@ -3237,6 +3274,7 @@ + ** features include but are not limited to the following: + ** <ul> + ** <li> The [PRAGMA writable_schema=ON] statement. ++** <li> The [PRAGMA journal_mode=OFF] statement. + ** <li> Writes to the [sqlite_dbpage] virtual table. + ** <li> Direct writes to [shadow tables]. + ** </ul> +@@ -3252,6 +3290,77 @@ + ** integer into which is written 0 or 1 to indicate whether the writable_schema + ** is enabled or disabled following this call. + ** </dd> ++** ++** [[SQLITE_DBCONFIG_LEGACY_ALTER_TABLE]] ++** <dt>SQLITE_DBCONFIG_LEGACY_ALTER_TABLE</dt> ++** <dd>The SQLITE_DBCONFIG_LEGACY_ALTER_TABLE option activates or deactivates ++** the legacy behavior of the [ALTER TABLE RENAME] command such it ++** behaves as it did prior to [version 3.24.0] (2018-06-04). See the ++** "Compatibility Notice" on the [ALTER TABLE RENAME documentation] for ++** additional information. This feature can also be turned on and off ++** using the [PRAGMA legacy_alter_table] statement. ++** </dd> ++** ++** [[SQLITE_DBCONFIG_DQS_DML]] ++** <dt>SQLITE_DBCONFIG_DQS_DML</td> ++** <dd>The SQLITE_DBCONFIG_DQS_DML option activates or deactivates ++** the legacy [double-quoted string literal] misfeature for DML statements ++** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The ++** default value of this setting is determined by the [-DSQLITE_DQS] ++** compile-time option. ++** </dd> ++** ++** [[SQLITE_DBCONFIG_DQS_DDL]] ++** <dt>SQLITE_DBCONFIG_DQS_DDL</td> ++** <dd>The SQLITE_DBCONFIG_DQS option activates or deactivates ++** the legacy [double-quoted string literal] misfeature for DDL statements, ++** such as CREATE TABLE and CREATE INDEX. The ++** default value of this setting is determined by the [-DSQLITE_DQS] ++** compile-time option. ++** </dd> ++** ++** [[SQLITE_DBCONFIG_TRUSTED_SCHEMA]] ++** <dt>SQLITE_DBCONFIG_TRUSTED_SCHEMA</td> ++** <dd>The SQLITE_DBCONFIG_TRUSTED_SCHEMA option tells SQLite to ++** assume that database schemas (the contents of the [sqlite_master] tables) ++** are untainted by malicious content. ++** When the SQLITE_DBCONFIG_TRUSTED_SCHEMA option is disabled, SQLite ++** takes additional defensive steps to protect the application from harm ++** including: ++** <ul> ++** <li> Prohibit the use of SQL functions inside triggers, views, ++** CHECK constraints, DEFAULT clauses, expression indexes, ++** partial indexes, or generated columns ++** unless those functions are tagged with [SQLITE_INNOCUOUS]. ++** <li> Prohibit the use of virtual tables inside of triggers or views ++** unless those virtual tables are tagged with [SQLITE_VTAB_INNOCUOUS]. ++** </ul> ++** This setting defaults to "on" for legacy compatibility, however ++** all applications are advised to turn it off if possible. This setting ++** can also be controlled using the [PRAGMA trusted_schema] statement. ++** </dd> ++** ++** [[SQLITE_DBCONFIG_LEGACY_FILE_FORMAT]] ++** <dt>SQLITE_DBCONFIG_LEGACY_FILE_FORMAT</td> ++** <dd>The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates ++** the legacy file format flag. When activated, this flag causes all newly ++** created database file to have a schema format version number (the 4-byte ++** integer found at offset 44 into the database header) of 1. This in turn ++** means that the resulting database file will be readable and writable by ++** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, ++** newly created databases are generally not understandable by SQLite versions ++** prior to 3.3.0 ([dateof:3.3.0]). As these words are written, there ++** is now scarcely any need to generated database files that are compatible ++** all the way back to version 3.0.0, and so this setting is of little ++** practical use, but is provided so that SQLite can continue to claim the ++** ability to generate new database files that are compatible with version ++** 3.0.0. ++** <p>Note that when the SQLITE_DBCONFIG_LEGACY_FILE_FORMAT setting is on, ++** the [VACUUM] command will fail with an obscure error when attempting to ++** process a table with generated columns and a descending index. This is ++** not considered a bug since SQLite versions 3.3.0 and earlier do not support ++** either generated columns or decending indexes. ++** </dd> + ** </dl> + */ + #define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ +@@ -3266,7 +3375,13 @@ + #define SQLITE_DBCONFIG_RESET_DATABASE 1009 /* int int* */ + #define SQLITE_DBCONFIG_DEFENSIVE 1010 /* int int* */ + #define SQLITE_DBCONFIG_WRITABLE_SCHEMA 1011 /* int int* */ +-#define SQLITE_DBCONFIG_MAX 1011 /* Largest DBCONFIG */ ++#define SQLITE_DBCONFIG_LEGACY_ALTER_TABLE 1012 /* int int* */ ++#define SQLITE_DBCONFIG_DQS_DML 1013 /* int int* */ ++#define SQLITE_DBCONFIG_DQS_DDL 1014 /* int int* */ ++#define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ ++#define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016 /* int int* */ ++#define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ ++#define SQLITE_DBCONFIG_MAX 1017 /* Largest DBCONFIG */ + + /* + ** CAPI3REF: Enable Or Disable Extended Result Codes +@@ -3472,7 +3587,7 @@ + ** ^The sqlite3_interrupt(D) call is in effect until all currently running + ** SQL statements on [database connection] D complete. ^Any new SQL statements + ** that are started after the sqlite3_interrupt() call and before the +-** running statements reaches zero are interrupted as if they had been ++** running statement count reaches zero are interrupted as if they had been + ** running prior to the sqlite3_interrupt() call. ^New SQL statements + ** that are started after the running statement count reaches zero are + ** not effected by the sqlite3_interrupt(). +@@ -3640,9 +3755,9 @@ + ** Cindy | 21 + ** </pre></blockquote> + ** +-** There are two column (M==2) and three rows (N==3). Thus the ++** There are two columns (M==2) and three rows (N==3). Thus the + ** result table has 8 entries. Suppose the result table is stored +-** in an array names azResult. Then azResult holds this content: ++** in an array named azResult. Then azResult holds this content: + ** + ** <blockquote><pre> + ** azResult[0] = "Name"; +@@ -3735,7 +3850,7 @@ + ** + ** The SQLite core uses these three routines for all of its own + ** internal memory allocation needs. "Core" in the previous sentence +-** does not include operating-system specific VFS implementation. The ++** does not include operating-system specific [VFS] implementation. The + ** Windows VFS uses native malloc() and free() for some operations. + ** + ** ^The sqlite3_malloc() routine returns a pointer to a block +@@ -3796,19 +3911,6 @@ + ** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time + ** option is used. + ** +-** In SQLite version 3.5.0 and 3.5.1, it was possible to define +-** the SQLITE_OMIT_MEMORY_ALLOCATION which would cause the built-in +-** implementation of these routines to be omitted. That capability +-** is no longer provided. Only built-in memory allocators can be used. +-** +-** Prior to SQLite version 3.7.10, the Windows OS interface layer called +-** the system malloc() and free() directly when converting +-** filenames between the UTF-8 encoding used by SQLite +-** and whatever filename encoding is used by the particular Windows +-** installation. Memory allocation errors were detected, but +-** they were reported back as [SQLITE_CANTOPEN] or +-** [SQLITE_IOERR] rather than [SQLITE_NOMEM]. +-** + ** The pointer arguments to [sqlite3_free()] and [sqlite3_realloc()] + ** must be either NULL or else pointers obtained from a prior + ** invocation of [sqlite3_malloc()] or [sqlite3_realloc()] that have +@@ -3857,7 +3959,7 @@ + ** SQLite contains a high-quality pseudo-random number generator (PRNG) used to + ** select random [ROWID | ROWIDs] when inserting new records into a table that + ** already uses the largest possible [ROWID]. The PRNG is also used for +-** the build-in random() and randomblob() SQL functions. This interface allows ++** the built-in random() and randomblob() SQL functions. This interface allows + ** applications to access the same PRNG for other purposes. + ** + ** ^A call to this routine stores N bytes of randomness into buffer P. +@@ -4231,10 +4333,8 @@ + ** The sqlite3_open_v2() interface works like sqlite3_open() + ** except that it accepts two additional parameters for additional control + ** over the new database connection. ^(The flags parameter to +-** sqlite3_open_v2() can take one of +-** the following three values, optionally combined with the +-** [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], [SQLITE_OPEN_SHAREDCACHE], +-** [SQLITE_OPEN_PRIVATECACHE], and/or [SQLITE_OPEN_URI] flags:)^ ++** sqlite3_open_v2() must include, at a minimum, one of the following ++** three flag combinations:)^ + ** + ** <dl> + ** ^(<dt>[SQLITE_OPEN_READONLY]</dt> +@@ -4252,23 +4352,51 @@ + ** sqlite3_open() and sqlite3_open16().</dd>)^ + ** </dl> + ** ++** In addition to the required flags, the following optional flags are ++** also supported: ++** ++** <dl> ++** ^(<dt>[SQLITE_OPEN_URI]</dt> ++** <dd>The filename can be interpreted as a URI if this flag is set.</dd>)^ ++** ++** ^(<dt>[SQLITE_OPEN_MEMORY]</dt> ++** <dd>The database will be opened as an in-memory database. The database ++** is named by the "filename" argument for the purposes of cache-sharing, ++** if shared cache mode is enabled, but the "filename" is otherwise ignored. ++** </dd>)^ ++** ++** ^(<dt>[SQLITE_OPEN_NOMUTEX]</dt> ++** <dd>The new database connection will use the "multi-thread" ++** [threading mode].)^ This means that separate threads are allowed ++** to use SQLite at the same time, as long as each thread is using ++** a different [database connection]. ++** ++** ^(<dt>[SQLITE_OPEN_FULLMUTEX]</dt> ++** <dd>The new database connection will use the "serialized" ++** [threading mode].)^ This means the multiple threads can safely ++** attempt to use the same database connection at the same time. ++** (Mutexes will block any actual concurrency, but in this mode ++** there is no harm in trying.) ++** ++** ^(<dt>[SQLITE_OPEN_SHAREDCACHE]</dt> ++** <dd>The database is opened [shared cache] enabled, overriding ++** the default shared cache setting provided by ++** [sqlite3_enable_shared_cache()].)^ ++** ++** ^(<dt>[SQLITE_OPEN_PRIVATECACHE]</dt> ++** <dd>The database is opened [shared cache] disabled, overriding ++** the default shared cache setting provided by ++** [sqlite3_enable_shared_cache()].)^ ++** ++** [[OPEN_NOFOLLOW]] ^(<dt>[SQLITE_OPEN_NOFOLLOW]</dt> ++** <dd>The database filename is not allowed to be a symbolic link</dd> ++** </dl>)^ ++** + ** If the 3rd parameter to sqlite3_open_v2() is not one of the +-** combinations shown above optionally combined with other ++** required combinations shown above optionally combined with other + ** [SQLITE_OPEN_READONLY | SQLITE_OPEN_* bits] + ** then the behavior is undefined. + ** +-** ^If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection +-** opens in the multi-thread [threading mode] as long as the single-thread +-** mode has not been set at compile-time or start-time. ^If the +-** [SQLITE_OPEN_FULLMUTEX] flag is set then the database connection opens +-** in the serialized [threading mode] unless single-thread was +-** previously selected at compile-time or start-time. +-** ^The [SQLITE_OPEN_SHAREDCACHE] flag causes the database connection to be +-** eligible to use [shared cache mode], regardless of whether or not shared +-** cache is enabled using [sqlite3_enable_shared_cache()]. ^The +-** [SQLITE_OPEN_PRIVATECACHE] flag causes the database connection to not +-** participate in [shared cache mode] even if it is enabled. +-** + ** ^The fourth parameter to sqlite3_open_v2() is the name of the + ** [sqlite3_vfs] object that defines the operating system interface that + ** the new database connection should use. ^If the fourth parameter is +@@ -4448,17 +4576,27 @@ + /* + ** CAPI3REF: Obtain Values For URI Parameters + ** +-** These are utility routines, useful to VFS implementations, that check +-** to see if a database file was a URI that contained a specific query ++** These are utility routines, useful to [VFS|custom VFS implementations], ++** that check if a database file was a URI that contained a specific query + ** parameter, and if so obtains the value of that query parameter. + ** +-** If F is the database filename pointer passed into the xOpen() method of +-** a VFS implementation when the flags parameter to xOpen() has one or +-** more of the [SQLITE_OPEN_URI] or [SQLITE_OPEN_MAIN_DB] bits set and +-** P is the name of the query parameter, then ++** The first parameter to these interfaces (hereafter referred to ++** as F) must be one of: ++** <ul> ++** <li> A database filename pointer created by the SQLite core and ++** passed into the xOpen() method of a VFS implemention, or ++** <li> A filename obtained from [sqlite3_db_filename()], or ++** <li> A new filename constructed using [sqlite3_create_filename()]. ++** </ul> ++** If the F parameter is not one of the above, then the behavior is ++** undefined and probably undesirable. Older versions of SQLite were ++** more tolerant of invalid F parameters than newer versions. ++** ++** If F is a suitable filename (as described in the previous paragraph) ++** and if P is the name of the query parameter, then + ** sqlite3_uri_parameter(F,P) returns the value of the P + ** parameter if it exists or a NULL pointer if P does not appear as a +-** query parameter on F. If P is a query parameter of F ++** query parameter on F. If P is a query parameter of F and it + ** has no explicit value, then sqlite3_uri_parameter(F,P) returns + ** a pointer to an empty string. + ** +@@ -4470,26 +4608,145 @@ + ** sqlite3_uri_boolean(F,P,B) routines returns false (0) if the value of + ** query parameter P is one of "no", "false", or "off" in any case or + ** if the value begins with a numeric zero. If P is not a query +-** parameter on F or if the value of P is does not match any of the ++** parameter on F or if the value of P does not match any of the + ** above, then sqlite3_uri_boolean(F,P,B) returns (B!=0). + ** + ** The sqlite3_uri_int64(F,P,D) routine converts the value of P into a + ** 64-bit signed integer and returns that integer, or D if P does not + ** exist. If the value of P is something other than an integer, then + ** zero is returned. ++** ++** The sqlite3_uri_key(F,N) returns a pointer to the name (not ++** the value) of the N-th query parameter for filename F, or a NULL ++** pointer if N is less than zero or greater than the number of query ++** parameters minus 1. The N value is zero-based so N should be 0 to obtain ++** the name of the first query parameter, 1 for the second parameter, and ++** so forth. + ** + ** If F is a NULL pointer, then sqlite3_uri_parameter(F,P) returns NULL and + ** sqlite3_uri_boolean(F,P,B) returns B. If F is not a NULL pointer and +-** is not a database file pathname pointer that SQLite passed into the xOpen +-** VFS method, then the behavior of this routine is undefined and probably +-** undesirable. ++** is not a database file pathname pointer that the SQLite core passed ++** into the xOpen VFS method, then the behavior of this routine is undefined ++** and probably undesirable. ++** ++** Beginning with SQLite [version 3.31.0] ([dateof:3.31.0]) the input F ++** parameter can also be the name of a rollback journal file or WAL file ++** in addition to the main database file. Prior to version 3.31.0, these ++** routines would only work if F was the name of the main database file. ++** When the F parameter is the name of the rollback journal or WAL file, ++** it has access to all the same query parameters as were found on the ++** main database file. + ** + ** See the [URI filename] documentation for additional information. + */ + SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); + SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); + SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); ++SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N); ++ ++/* ++** CAPI3REF: Translate filenames ++** ++** These routines are available to [VFS|custom VFS implementations] for ++** translating filenames between the main database file, the journal file, ++** and the WAL file. ++** ++** If F is the name of an sqlite database file, journal file, or WAL file ++** passed by the SQLite core into the VFS, then sqlite3_filename_database(F) ++** returns the name of the corresponding database file. ++** ++** If F is the name of an sqlite database file, journal file, or WAL file ++** passed by the SQLite core into the VFS, or if F is a database filename ++** obtained from [sqlite3_db_filename()], then sqlite3_filename_journal(F) ++** returns the name of the corresponding rollback journal file. ++** ++** If F is the name of an sqlite database file, journal file, or WAL file ++** that was passed by the SQLite core into the VFS, or if F is a database ++** filename obtained from [sqlite3_db_filename()], then ++** sqlite3_filename_wal(F) returns the name of the corresponding ++** WAL file. ++** ++** In all of the above, if F is not the name of a database, journal or WAL ++** filename passed into the VFS from the SQLite core and F is not the ++** return value from [sqlite3_db_filename()], then the result is ++** undefined and is likely a memory access violation. ++*/ ++SQLITE_API const char *sqlite3_filename_database(const char*); ++SQLITE_API const char *sqlite3_filename_journal(const char*); ++SQLITE_API const char *sqlite3_filename_wal(const char*); ++ ++/* ++** CAPI3REF: Database File Corresponding To A Journal ++** ++** ^If X is the name of a rollback or WAL-mode journal file that is ++** passed into the xOpen method of [sqlite3_vfs], then ++** sqlite3_database_file_object(X) returns a pointer to the [sqlite3_file] ++** object that represents the main database file. ++** ++** This routine is intended for use in custom [VFS] implementations ++** only. It is not a general-purpose interface. ++** The argument sqlite3_file_object(X) must be a filename pointer that ++** has been passed into [sqlite3_vfs].xOpen method where the ++** flags parameter to xOpen contains one of the bits ++** [SQLITE_OPEN_MAIN_JOURNAL] or [SQLITE_OPEN_WAL]. Any other use ++** of this routine results in undefined and probably undesirable ++** behavior. ++*/ ++SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*); + ++/* ++** CAPI3REF: Create and Destroy VFS Filenames ++** ++** These interfces are provided for use by [VFS shim] implementations and ++** are not useful outside of that context. ++** ++** The sqlite3_create_filename(D,J,W,N,P) allocates memory to hold a version of ++** database filename D with corresponding journal file J and WAL file W and ++** with N URI parameters key/values pairs in the array P. The result from ++** sqlite3_create_filename(D,J,W,N,P) is a pointer to a database filename that ++** is safe to pass to routines like: ++** <ul> ++** <li> [sqlite3_uri_parameter()], ++** <li> [sqlite3_uri_boolean()], ++** <li> [sqlite3_uri_int64()], ++** <li> [sqlite3_uri_key()], ++** <li> [sqlite3_filename_database()], ++** <li> [sqlite3_filename_journal()], or ++** <li> [sqlite3_filename_wal()]. ++** </ul> ++** If a memory allocation error occurs, sqlite3_create_filename() might ++** return a NULL pointer. The memory obtained from sqlite3_create_filename(X) ++** must be released by a corresponding call to sqlite3_free_filename(Y). ++** ++** The P parameter in sqlite3_create_filename(D,J,W,N,P) should be an array ++** of 2*N pointers to strings. Each pair of pointers in this array corresponds ++** to a key and value for a query parameter. The P parameter may be a NULL ++** pointer if N is zero. None of the 2*N pointers in the P array may be ++** NULL pointers and key pointers should not be empty strings. ++** None of the D, J, or W parameters to sqlite3_create_filename(D,J,W,N,P) may ++** be NULL pointers, though they can be empty strings. ++** ++** The sqlite3_free_filename(Y) routine releases a memory allocation ++** previously obtained from sqlite3_create_filename(). Invoking ++** sqlite3_free_filename(Y) where Y is a NULL pointer is a harmless no-op. ++** ++** If the Y parameter to sqlite3_free_filename(Y) is anything other ++** than a NULL pointer or a pointer previously acquired from ++** sqlite3_create_filename(), then bad things such as heap ++** corruption or segfaults may occur. The value Y should be ++** used again after sqlite3_free_filename(Y) has been called. This means ++** that if the [sqlite3_vfs.xOpen()] method of a VFS has been called using Y, ++** then the corresponding [sqlite3_module.xClose() method should also be ++** invoked prior to calling sqlite3_free_filename(Y). ++*/ ++SQLITE_API char *sqlite3_create_filename( ++ const char *zDatabase, ++ const char *zJournal, ++ const char *zWal, ++ int nParam, ++ const char **azParam ++); ++SQLITE_API void sqlite3_free_filename(char*); + + /* + ** CAPI3REF: Error Codes And Messages +@@ -4807,15 +5064,15 @@ + ** </li> + ** + ** <li> +-** ^If the specific value bound to [parameter | host parameter] in the ++** ^If the specific value bound to a [parameter | host parameter] in the + ** WHERE clause might influence the choice of query plan for a statement, + ** then the statement will be automatically recompiled, as if there had been +-** a schema change, on the first [sqlite3_step()] call following any change ++** a schema change, on the first [sqlite3_step()] call following any change + ** to the [sqlite3_bind_text | bindings] of that [parameter]. +-** ^The specific value of WHERE-clause [parameter] might influence the ++** ^The specific value of a WHERE-clause [parameter] might influence the + ** choice of query plan if the parameter is the left-hand side of a [LIKE] + ** or [GLOB] operator or if the parameter is compared to an indexed column +-** and the [SQLITE_ENABLE_STAT3] compile-time option is enabled. ++** and the [SQLITE_ENABLE_STAT4] compile-time option is enabled. + ** </li> + ** </ol> + ** +@@ -5072,12 +5329,30 @@ + ** [sqlite3_bind_parameter_index()] API if desired. ^The index + ** for "?NNN" parameters is the value of NNN. + ** ^The NNN value must be between 1 and the [sqlite3_limit()] +-** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 999). ++** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 32766). + ** + ** ^The third argument is the value to bind to the parameter. + ** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16() + ** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter + ** is ignored and the end result is the same as sqlite3_bind_null(). ++** ^If the third parameter to sqlite3_bind_text() is not NULL, then ++** it should be a pointer to well-formed UTF8 text. ++** ^If the third parameter to sqlite3_bind_text16() is not NULL, then ++** it should be a pointer to well-formed UTF16 text. ++** ^If the third parameter to sqlite3_bind_text64() is not NULL, then ++** it should be a pointer to a well-formed unicode string that is ++** either UTF8 if the sixth parameter is SQLITE_UTF8, or UTF16 ++** otherwise. ++** ++** [[byte-order determination rules]] ^The byte-order of ++** UTF16 input text is determined by the byte-order mark (BOM, U+FEFF) ++** found in first character, which is removed, or in the absence of a BOM ++** the byte order is the native byte order of the host ++** machine for sqlite3_bind_text16() or the byte order specified in ++** the 6th parameter for sqlite3_bind_text64().)^ ++** ^If UTF16 input text contains invalid unicode ++** characters, then SQLite might change those invalid characters ++** into the unicode replacement character: U+FFFD. + ** + ** ^(In those routines that have a fourth argument, its value is the + ** number of bytes in the parameter. To be clear: the value is the +@@ -5091,7 +5366,7 @@ + ** or sqlite3_bind_text16() or sqlite3_bind_text64() then + ** that parameter must be the byte offset + ** where the NUL terminator would occur assuming the string were NUL +-** terminated. If any NUL characters occur at byte offsets less than ++** terminated. If any NUL characters occurs at byte offsets less than + ** the value of the fourth parameter then the resulting string value will + ** contain embedded NULs. The result of expressions involving strings + ** with embedded NULs is undefined. +@@ -5321,7 +5596,7 @@ + ** + ** ^If the Nth column returned by the statement is an expression or + ** subquery and is not a column value, then all of these functions return +-** NULL. ^These routine might also return NULL if a memory allocation error ++** NULL. ^These routines might also return NULL if a memory allocation error + ** occurs. ^Otherwise, they return the name of the attached database, table, + ** or column that query result column was extracted from. + ** +@@ -5331,10 +5606,6 @@ + ** ^These APIs are only available if the library was compiled with the + ** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol. + ** +-** If two or more threads call one or more of these routines against the same +-** prepared statement and column at the same time then the results are +-** undefined. +-** + ** If two or more threads call one or more + ** [sqlite3_column_database_name | column metadata interfaces] + ** for the same [prepared statement] and result column +@@ -5471,7 +5742,7 @@ + ** ^The sqlite3_data_count(P) interface returns the number of columns in the + ** current row of the result set of [prepared statement] P. + ** ^If prepared statement P does not have results ready to return +-** (via calls to the [sqlite3_column_int | sqlite3_column_*()] of ++** (via calls to the [sqlite3_column_int | sqlite3_column()] family of + ** interfaces) then sqlite3_data_count(P) returns 0. + ** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer. + ** ^The sqlite3_data_count(P) routine returns 0 if the previous call to +@@ -5795,8 +6066,6 @@ + /* + ** CAPI3REF: Create Or Redefine SQL Functions + ** KEYWORDS: {function creation routines} +-** KEYWORDS: {application-defined SQL function} +-** KEYWORDS: {application-defined SQL functions} + ** METHOD: sqlite3 + ** + ** ^These functions (collectively known as "function creation routines") +@@ -5850,6 +6119,23 @@ + ** perform additional optimizations on deterministic functions, so use + ** of the [SQLITE_DETERMINISTIC] flag is recommended where possible. + ** ++** ^The fourth parameter may also optionally include the [SQLITE_DIRECTONLY] ++** flag, which if present prevents the function from being invoked from ++** within VIEWs, TRIGGERs, CHECK constraints, generated column expressions, ++** index expressions, or the WHERE clause of partial indexes. ++** ++** <span style="background-color:#ffff90;"> ++** For best security, the [SQLITE_DIRECTONLY] flag is recommended for ++** all application-defined SQL functions that do not need to be ++** used inside of triggers, view, CHECK constraints, or other elements of ++** the database schema. This flags is especially recommended for SQL ++** functions that have side effects or reveal internal application state. ++** Without this flag, an attacker might be able to modify the schema of ++** a database file to include invocations of the function with parameters ++** chosen by the attacker, which the application will then execute when ++** the database file is opened and read. ++** </span> ++** + ** ^(The fifth parameter is an arbitrary pointer. The implementation of the + ** function can gain access to this pointer using [sqlite3_user_data()].)^ + ** +@@ -5966,8 +6252,68 @@ + ** [SQLITE_UTF8 | preferred text encoding] as the fourth argument + ** to [sqlite3_create_function()], [sqlite3_create_function16()], or + ** [sqlite3_create_function_v2()]. ++** ++** <dl> ++** [[SQLITE_DETERMINISTIC]] <dt>SQLITE_DETERMINISTIC</dt><dd> ++** The SQLITE_DETERMINISTIC flag means that the new function always gives ++** the same output when the input parameters are the same. ++** The [abs|abs() function] is deterministic, for example, but ++** [randomblob|randomblob()] is not. Functions must ++** be deterministic in order to be used in certain contexts such as ++** with the WHERE clause of [partial indexes] or in [generated columns]. ++** SQLite might also optimize deterministic functions by factoring them ++** out of inner loops. ++** </dd> ++** ++** [[SQLITE_DIRECTONLY]] <dt>SQLITE_DIRECTONLY</dt><dd> ++** The SQLITE_DIRECTONLY flag means that the function may only be invoked ++** from top-level SQL, and cannot be used in VIEWs or TRIGGERs nor in ++** schema structures such as [CHECK constraints], [DEFAULT clauses], ++** [expression indexes], [partial indexes], or [generated columns]. ++** The SQLITE_DIRECTONLY flags is a security feature which is recommended ++** for all [application-defined SQL functions], and especially for functions ++** that have side-effects or that could potentially leak sensitive ++** information. ++** </dd> ++** ++** [[SQLITE_INNOCUOUS]] <dt>SQLITE_INNOCUOUS</dt><dd> ++** The SQLITE_INNOCUOUS flag means that the function is unlikely ++** to cause problems even if misused. An innocuous function should have ++** no side effects and should not depend on any values other than its ++** input parameters. The [abs|abs() function] is an example of an ++** innocuous function. ++** The [load_extension() SQL function] is not innocuous because of its ++** side effects. ++** <p> SQLITE_INNOCUOUS is similar to SQLITE_DETERMINISTIC, but is not ++** exactly the same. The [random|random() function] is an example of a ++** function that is innocuous but not deterministic. ++** <p>Some heightened security settings ++** ([SQLITE_DBCONFIG_TRUSTED_SCHEMA] and [PRAGMA trusted_schema=OFF]) ++** disable the use of SQL functions inside views and triggers and in ++** schema structures such as [CHECK constraints], [DEFAULT clauses], ++** [expression indexes], [partial indexes], and [generated columns] unless ++** the function is tagged with SQLITE_INNOCUOUS. Most built-in functions ++** are innocuous. Developers are advised to avoid using the ++** SQLITE_INNOCUOUS flag for application-defined functions unless the ++** function has been carefully audited and found to be free of potentially ++** security-adverse side-effects and information-leaks. ++** </dd> ++** ++** [[SQLITE_SUBTYPE]] <dt>SQLITE_SUBTYPE</dt><dd> ++** The SQLITE_SUBTYPE flag indicates to SQLite that a function may call ++** [sqlite3_value_subtype()] to inspect the sub-types of its arguments. ++** Specifying this flag makes no difference for scalar or aggregate user ++** functions. However, if it is not specified for a user-defined window ++** function, then any sub-types belonging to arguments passed to the window ++** function may be discarded before the window function is called (i.e. ++** sqlite3_value_subtype() will always return 0). ++** </dd> ++** </dl> + */ +-#define SQLITE_DETERMINISTIC 0x800 ++#define SQLITE_DETERMINISTIC 0x000000800 ++#define SQLITE_DIRECTONLY 0x000080000 ++#define SQLITE_SUBTYPE 0x000100000 ++#define SQLITE_INNOCUOUS 0x000200000 + + /* + ** CAPI3REF: Deprecated Functions +@@ -6026,8 +6372,8 @@ + ** + ** These routines extract type, size, and content information from + ** [protected sqlite3_value] objects. Protected sqlite3_value objects +-** are used to pass parameter information into implementation of +-** [application-defined SQL functions] and [virtual tables]. ++** are used to pass parameter information into the functions that ++** implement [application-defined SQL functions] and [virtual tables]. + ** + ** These routines work only with [protected sqlite3_value] objects. + ** Any attempt to use these routines on an [unprotected sqlite3_value] +@@ -6084,7 +6430,7 @@ + ** ^The sqlite3_value_frombind(X) interface returns non-zero if the + ** value X originated from one of the [sqlite3_bind_int|sqlite3_bind()] + ** interfaces. ^If X comes from an SQL literal value, or a table column, +-** and expression, then sqlite3_value_frombind(X) returns zero. ++** or an expression, then sqlite3_value_frombind(X) returns zero. + ** + ** Please pay particular attention to the fact that the pointer returned + ** from [sqlite3_value_blob()], [sqlite3_value_text()], or +@@ -6170,8 +6516,8 @@ + ** routine to allocate memory for storing their state. + ** + ** ^The first time the sqlite3_aggregate_context(C,N) routine is called +-** for a particular aggregate function, SQLite +-** allocates N of memory, zeroes out that memory, and returns a pointer ++** for a particular aggregate function, SQLite allocates ++** N bytes of memory, zeroes out that memory, and returns a pointer + ** to the new memory. ^On second and subsequent calls to + ** sqlite3_aggregate_context() for the same aggregate function instance, + ** the same buffer is returned. Sqlite3_aggregate_context() is normally +@@ -6188,7 +6534,7 @@ + ** + ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is + ** determined by the N parameter on first successful call. Changing the +-** value of N in subsequent call to sqlite3_aggregate_context() within ++** value of N in any subsequent call to sqlite3_aggregate_context() within + ** the same aggregate function instance will not resize the memory + ** allocation.)^ Within the xFinal callback, it is customary to set + ** N=0 in calls to sqlite3_aggregate_context(C,N) so that no +@@ -6345,8 +6691,9 @@ + ** 2nd parameter of sqlite3_result_error() or sqlite3_result_error16() + ** as the text of an error message. ^SQLite interprets the error + ** message string from sqlite3_result_error() as UTF-8. ^SQLite +-** interprets the string from sqlite3_result_error16() as UTF-16 in native +-** byte order. ^If the third parameter to sqlite3_result_error() ++** interprets the string from sqlite3_result_error16() as UTF-16 using ++** the same [byte-order determination rules] as [sqlite3_bind_text16()]. ++** ^If the third parameter to sqlite3_result_error() + ** or sqlite3_result_error16() is negative then SQLite takes as the error + ** message all text up through the first zero character. + ** ^If the third parameter to sqlite3_result_error() or +@@ -6414,6 +6761,25 @@ + ** then SQLite makes a copy of the result into space obtained + ** from [sqlite3_malloc()] before it returns. + ** ++** ^For the sqlite3_result_text16(), sqlite3_result_text16le(), and ++** sqlite3_result_text16be() routines, and for sqlite3_result_text64() ++** when the encoding is not UTF8, if the input UTF16 begins with a ++** byte-order mark (BOM, U+FEFF) then the BOM is removed from the ++** string and the rest of the string is interpreted according to the ++** byte-order specified by the BOM. ^The byte-order specified by ++** the BOM at the beginning of the text overrides the byte-order ++** specified by the interface procedure. ^So, for example, if ++** sqlite3_result_text16le() is invoked with text that begins ++** with bytes 0xfe, 0xff (a big-endian byte-order mark) then the ++** first two bytes of input are skipped and the remaining input ++** is interpreted as UTF16BE text. ++** ++** ^For UTF16 input text to the sqlite3_result_text16(), ++** sqlite3_result_text16be(), sqlite3_result_text16le(), and ++** sqlite3_result_text64() routines, if the text contains invalid ++** UTF16 characters, the invalid characters might be converted ++** into the unicode replacement character, U+FFFD. ++** + ** ^The sqlite3_result_value() interface sets the result of + ** the application-defined function to be a copy of the + ** [unprotected sqlite3_value] object specified by the 2nd parameter. ^The +@@ -6499,7 +6865,7 @@ + ** <li> [SQLITE_UTF16_ALIGNED]. + ** </ul>)^ + ** ^The eTextRep argument determines the encoding of strings passed +-** to the collating function callback, xCallback. ++** to the collating function callback, xCompare. + ** ^The [SQLITE_UTF16] and [SQLITE_UTF16_ALIGNED] values for eTextRep + ** force strings to be UTF16 with native byte order. + ** ^The [SQLITE_UTF16_ALIGNED] value for eTextRep forces strings to begin +@@ -6508,18 +6874,19 @@ + ** ^The fourth argument, pArg, is an application data pointer that is passed + ** through as the first argument to the collating function callback. + ** +-** ^The fifth argument, xCallback, is a pointer to the collating function. ++** ^The fifth argument, xCompare, is a pointer to the collating function. + ** ^Multiple collating functions can be registered using the same name but + ** with different eTextRep parameters and SQLite will use whichever + ** function requires the least amount of data transformation. +-** ^If the xCallback argument is NULL then the collating function is ++** ^If the xCompare argument is NULL then the collating function is + ** deleted. ^When all collating functions having the same name are deleted, + ** that collation is no longer usable. + ** + ** ^The collating function callback is invoked with a copy of the pArg + ** application data pointer and with two strings in the encoding specified +-** by the eTextRep argument. The collating function must return an +-** integer that is negative, zero, or positive ++** by the eTextRep argument. The two integer parameters to the collating ++** function callback are the length of the two strings, in bytes. The collating ++** function must return an integer that is negative, zero, or positive + ** if the first string is less than, equal to, or greater than the second, + ** respectively. A collating function must always return the same answer + ** given the same inputs. If two or more collating functions are registered +@@ -6536,7 +6903,7 @@ + ** </ol> + ** + ** If a collating function fails any of the above constraints and that +-** collating function is registered and used, then the behavior of SQLite ++** collating function is registered and used, then the behavior of SQLite + ** is undefined. + ** + ** ^The sqlite3_create_collation_v2() works like sqlite3_create_collation() +@@ -6618,51 +6985,6 @@ + void(*)(void*,sqlite3*,int eTextRep,const void*) + ); + +-#ifdef SQLITE_HAS_CODEC +-/* +-** Specify the key for an encrypted database. This routine should be +-** called right after sqlite3_open(). +-** +-** The code to implement this API is not available in the public release +-** of SQLite. +-*/ +-SQLITE_API int sqlite3_key( +- sqlite3 *db, /* Database to be rekeyed */ +- const void *pKey, int nKey /* The key */ +-); +-SQLITE_API int sqlite3_key_v2( +- sqlite3 *db, /* Database to be rekeyed */ +- const char *zDbName, /* Name of the database */ +- const void *pKey, int nKey /* The key */ +-); +- +-/* +-** Change the key on an open database. If the current database is not +-** encrypted, this routine will encrypt it. If pNew==0 or nNew==0, the +-** database is decrypted. +-** +-** The code to implement this API is not available in the public release +-** of SQLite. +-*/ +-SQLITE_API int sqlite3_rekey( +- sqlite3 *db, /* Database to be rekeyed */ +- const void *pKey, int nKey /* The new key */ +-); +-SQLITE_API int sqlite3_rekey_v2( +- sqlite3 *db, /* Database to be rekeyed */ +- const char *zDbName, /* Name of the database */ +- const void *pKey, int nKey /* The new key */ +-); +- +-/* +-** Specify the activation key for a SEE database. Unless +-** activated, none of the SEE routines will work. +-*/ +-SQLITE_API void sqlite3_activate_see( +- const char *zPassPhrase /* Activation phrase */ +-); +-#endif +- + #ifdef SQLITE_ENABLE_CEROD + /* + ** Specify the activation key for a CEROD database. Unless +@@ -6863,16 +7185,31 @@ + ** CAPI3REF: Return The Filename For A Database Connection + ** METHOD: sqlite3 + ** +-** ^The sqlite3_db_filename(D,N) interface returns a pointer to a filename +-** associated with database N of connection D. ^The main database file +-** has the name "main". If there is no attached database N on the database ++** ^The sqlite3_db_filename(D,N) interface returns a pointer to the filename ++** associated with database N of connection D. ++** ^If there is no attached database N on the database + ** connection D, or if database N is a temporary or in-memory database, then + ** this function will return either a NULL pointer or an empty string. + ** ++** ^The string value returned by this routine is owned and managed by ++** the database connection. ^The value will be valid until the database N ++** is [DETACH]-ed or until the database connection closes. ++** + ** ^The filename returned by this function is the output of the + ** xFullPathname method of the [VFS]. ^In other words, the filename + ** will be an absolute pathname, even if the filename used + ** to open the database originally was a URI or relative pathname. ++** ++** If the filename pointer returned by this routine is not NULL, then it ++** can be used as the filename input parameter to these routines: ++** <ul> ++** <li> [sqlite3_uri_parameter()] ++** <li> [sqlite3_uri_boolean()] ++** <li> [sqlite3_uri_int64()] ++** <li> [sqlite3_filename_database()] ++** <li> [sqlite3_filename_journal()] ++** <li> [sqlite3_filename_wal()] ++** </ul> + */ + SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); + +@@ -7022,15 +7359,19 @@ + ** + ** ^(The cache sharing mode set by this interface effects all subsequent + ** calls to [sqlite3_open()], [sqlite3_open_v2()], and [sqlite3_open16()]. +-** Existing database connections continue use the sharing mode ++** Existing database connections continue to use the sharing mode + ** that was in effect at the time they were opened.)^ + ** + ** ^(This routine returns [SQLITE_OK] if shared cache was enabled or disabled + ** successfully. An [error code] is returned otherwise.)^ + ** +-** ^Shared cache is disabled by default. But this might change in +-** future releases of SQLite. Applications that care about shared +-** cache setting should set it explicitly. ++** ^Shared cache is disabled by default. It is recommended that it stay ++** that way. In other words, do not use this routine. This interface ++** continues to be provided for historical compatibility, but its use is ++** discouraged. Any use of shared cache is discouraged. If shared cache ++** must be used, it is recommended that shared cache only be enabled for ++** individual database connections using the [sqlite3_open_v2()] interface ++** with the [SQLITE_OPEN_SHAREDCACHE] flag. + ** + ** Note: This method is disabled on MacOS X 10.7 and iOS version 5.0 + ** and will always return SQLITE_MISUSE. On those systems, +@@ -7077,6 +7418,9 @@ + /* + ** CAPI3REF: Impose A Limit On Heap Size + ** ++** These interfaces impose limits on the amount of heap memory that will be ++** by all database connections within a single process. ++** + ** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the + ** soft limit on the amount of heap memory that may be allocated by SQLite. + ** ^SQLite strives to keep heap memory utilization below the soft heap +@@ -7087,20 +7431,41 @@ + ** an [SQLITE_NOMEM] error. In other words, the soft heap limit + ** is advisory only. + ** +-** ^The return value from sqlite3_soft_heap_limit64() is the size of +-** the soft heap limit prior to the call, or negative in the case of an +-** error. ^If the argument N is negative +-** then no change is made to the soft heap limit. Hence, the current +-** size of the soft heap limit can be determined by invoking +-** sqlite3_soft_heap_limit64() with a negative argument. +-** +-** ^If the argument N is zero then the soft heap limit is disabled. ++** ^The sqlite3_hard_heap_limit64(N) interface sets a hard upper bound of ++** N bytes on the amount of memory that will be allocated. ^The ++** sqlite3_hard_heap_limit64(N) interface is similar to ++** sqlite3_soft_heap_limit64(N) except that memory allocations will fail ++** when the hard heap limit is reached. + ** +-** ^(The soft heap limit is not enforced in the current implementation ++** ^The return value from both sqlite3_soft_heap_limit64() and ++** sqlite3_hard_heap_limit64() is the size of ++** the heap limit prior to the call, or negative in the case of an ++** error. ^If the argument N is negative ++** then no change is made to the heap limit. Hence, the current ++** size of heap limits can be determined by invoking ++** sqlite3_soft_heap_limit64(-1) or sqlite3_hard_heap_limit(-1). ++** ++** ^Setting the heap limits to zero disables the heap limiter mechanism. ++** ++** ^The soft heap limit may not be greater than the hard heap limit. ++** ^If the hard heap limit is enabled and if sqlite3_soft_heap_limit(N) ++** is invoked with a value of N that is greater than the hard heap limit, ++** the the soft heap limit is set to the value of the hard heap limit. ++** ^The soft heap limit is automatically enabled whenever the hard heap ++** limit is enabled. ^When sqlite3_hard_heap_limit64(N) is invoked and ++** the soft heap limit is outside the range of 1..N, then the soft heap ++** limit is set to N. ^Invoking sqlite3_soft_heap_limit64(0) when the ++** hard heap limit is enabled makes the soft heap limit equal to the ++** hard heap limit. ++** ++** The memory allocation limits can also be adjusted using ++** [PRAGMA soft_heap_limit] and [PRAGMA hard_heap_limit]. ++** ++** ^(The heap limits are not enforced in the current implementation + ** if one or more of following conditions are true: + ** + ** <ul> +-** <li> The soft heap limit is set to zero. ++** <li> The limit value is set to zero. + ** <li> Memory accounting is disabled using a combination of the + ** [sqlite3_config]([SQLITE_CONFIG_MEMSTATUS],...) start-time option and + ** the [SQLITE_DEFAULT_MEMSTATUS] compile-time option. +@@ -7111,21 +7476,11 @@ + ** from the heap. + ** </ul>)^ + ** +-** Beginning with SQLite [version 3.7.3] ([dateof:3.7.3]), +-** the soft heap limit is enforced +-** regardless of whether or not the [SQLITE_ENABLE_MEMORY_MANAGEMENT] +-** compile-time option is invoked. With [SQLITE_ENABLE_MEMORY_MANAGEMENT], +-** the soft heap limit is enforced on every memory allocation. Without +-** [SQLITE_ENABLE_MEMORY_MANAGEMENT], the soft heap limit is only enforced +-** when memory is allocated by the page cache. Testing suggests that because +-** the page cache is the predominate memory user in SQLite, most +-** applications will achieve adequate soft heap limit enforcement without +-** the use of [SQLITE_ENABLE_MEMORY_MANAGEMENT]. +-** +-** The circumstances under which SQLite will enforce the soft heap limit may ++** The circumstances under which SQLite will enforce the heap limits may + ** changes in future releases of SQLite. + */ + SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); ++SQLITE_API sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 N); + + /* + ** CAPI3REF: Deprecated Soft Heap Limit Interface +@@ -7149,7 +7504,7 @@ + ** interface returns SQLITE_OK and fills in the non-NULL pointers in + ** the final five arguments with appropriate values if the specified + ** column exists. ^The sqlite3_table_column_metadata() interface returns +-** SQLITE_ERROR and if the specified column does not exist. ++** SQLITE_ERROR if the specified column does not exist. + ** ^If the column-name parameter to sqlite3_table_column_metadata() is a + ** NULL pointer, then this routine simply checks for the existence of the + ** table and returns SQLITE_OK if the table exists and SQLITE_ERROR if it +@@ -7291,7 +7646,7 @@ + ** to enable or disable only the C-API.)^ + ** + ** <b>Security warning:</b> It is recommended that extension loading +-** be disabled using the [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method ++** be enabled using the [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method + ** rather than this interface, so the [load_extension()] SQL function + ** remains disabled. This will prevent SQL injections from giving attackers + ** access to extension loading capabilities. +@@ -7378,7 +7733,7 @@ + ** KEYWORDS: sqlite3_module {virtual table module} + ** + ** This structure, sometimes called a "virtual table module", +-** defines the implementation of a [virtual tables]. ++** defines the implementation of a [virtual table]. + ** This structure consists mostly of methods for the module. + ** + ** ^A virtual table module is created by filling in a persistent +@@ -7475,7 +7830,13 @@ + ** the right-hand side of the corresponding aConstraint[] is evaluated + ** and becomes the argvIndex-th entry in argv. ^(If aConstraintUsage[].omit + ** is true, then the constraint is assumed to be fully handled by the +-** virtual table and is not checked again by SQLite.)^ ++** virtual table and might not be checked again by the byte code.)^ ^(The ++** aConstraintUsage[].omit flag is an optimization hint. When the omit flag ++** is left in its default setting of false, the constraint will always be ++** checked separately in byte code. If the omit flag is change to true, then ++** the constraint may or may not be checked in byte code. In other words, ++** when the omit flag is true there is no guarantee that the constraint will ++** not be checked again using byte code.)^ + ** + ** ^The idxNum and idxPtr values are recorded and passed into the + ** [xFilter] method. +@@ -7515,7 +7876,7 @@ + ** If a virtual table extension is + ** used with an SQLite version earlier than 3.8.2, the results of attempting + ** to read or write the estimatedRows field are undefined (but are likely +-** to included crashing the application). The estimatedRows field should ++** to include crashing the application). The estimatedRows field should + ** therefore only be used if [sqlite3_libversion_number()] returns a + ** value greater than or equal to 3008002. Similarly, the idxFlags field + ** was added for [version 3.9.0] ([dateof:3.9.0]). +@@ -7567,7 +7928,7 @@ + /* + ** CAPI3REF: Virtual Table Constraint Operator Codes + ** +-** These macros defined the allowed values for the ++** These macros define the allowed values for the + ** [sqlite3_index_info].aConstraint[].op field. Each value represents + ** an operator that is part of a constraint term in the wHERE clause of + ** a query that uses a [virtual table]. +@@ -7613,6 +7974,12 @@ + ** ^The sqlite3_create_module() + ** interface is equivalent to sqlite3_create_module_v2() with a NULL + ** destructor. ++** ++** ^If the third parameter (the pointer to the sqlite3_module object) is ++** NULL then no new module is create and any existing modules with the ++** same name are dropped. ++** ++** See also: [sqlite3_drop_modules()] + */ + SQLITE_API int sqlite3_create_module( + sqlite3 *db, /* SQLite connection to register module with */ +@@ -7628,6 +7995,23 @@ + void(*xDestroy)(void*) /* Module destructor function */ + ); + ++/* ++** CAPI3REF: Remove Unnecessary Virtual Table Implementations ++** METHOD: sqlite3 ++** ++** ^The sqlite3_drop_modules(D,L) interface removes all virtual ++** table modules from database connection D except those named on list L. ++** The L parameter must be either NULL or a pointer to an array of pointers ++** to strings where the array is terminated by a single NULL pointer. ++** ^If the L parameter is NULL, then all virtual table modules are removed. ++** ++** See also: [sqlite3_create_module()] ++*/ ++SQLITE_API int sqlite3_drop_modules( ++ sqlite3 *db, /* Remove modules from this connection */ ++ const char **azKeep /* Except, do not remove the ones named here */ ++); ++ + /* + ** CAPI3REF: Virtual Table Instance Object + ** KEYWORDS: sqlite3_vtab +@@ -8154,7 +8538,7 @@ + ** The only difference is that the public sqlite3_XXX functions enumerated + ** above silently ignore any invocations that pass a NULL pointer instead + ** of a valid mutex handle. The implementations of the methods defined +-** by this structure are not required to handle this case, the results ++** by this structure are not required to handle this case. The results + ** of passing a NULL pointer instead of a valid mutex handle are undefined + ** (i.e. it is acceptable to provide an implementation that segfaults if + ** it is passed a NULL pointer). +@@ -8336,14 +8720,14 @@ + #define SQLITE_TESTCTRL_FIRST 5 + #define SQLITE_TESTCTRL_PRNG_SAVE 5 + #define SQLITE_TESTCTRL_PRNG_RESTORE 6 +-#define SQLITE_TESTCTRL_PRNG_RESET 7 ++#define SQLITE_TESTCTRL_PRNG_RESET 7 /* NOT USED */ + #define SQLITE_TESTCTRL_BITVEC_TEST 8 + #define SQLITE_TESTCTRL_FAULT_INSTALL 9 + #define SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS 10 + #define SQLITE_TESTCTRL_PENDING_BYTE 11 + #define SQLITE_TESTCTRL_ASSERT 12 + #define SQLITE_TESTCTRL_ALWAYS 13 +-#define SQLITE_TESTCTRL_RESERVE 14 ++#define SQLITE_TESTCTRL_RESERVE 14 /* NOT USED */ + #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 + #define SQLITE_TESTCTRL_ISKEYWORD 16 /* NOT USED */ + #define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */ +@@ -8358,7 +8742,10 @@ + #define SQLITE_TESTCTRL_SORTER_MMAP 24 + #define SQLITE_TESTCTRL_IMPOSTER 25 + #define SQLITE_TESTCTRL_PARSER_COVERAGE 26 +-#define SQLITE_TESTCTRL_LAST 26 /* Largest TESTCTRL */ ++#define SQLITE_TESTCTRL_RESULT_INTREAL 27 ++#define SQLITE_TESTCTRL_PRNG_SEED 28 ++#define SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS 29 ++#define SQLITE_TESTCTRL_LAST 29 /* Largest TESTCTRL */ + + /* + ** CAPI3REF: SQL Keyword Checking +@@ -8624,7 +9011,7 @@ + ** + ** [[SQLITE_STATUS_PAGECACHE_SIZE]] ^(<dt>SQLITE_STATUS_PAGECACHE_SIZE</dt> + ** <dd>This parameter records the largest memory allocation request +-** handed to [pagecache memory allocator]. Only the value returned in the ++** handed to the [pagecache memory allocator]. Only the value returned in the + ** *pHighwater parameter to [sqlite3_status()] is of interest. + ** The value written into the *pCurrent parameter is undefined.</dd>)^ + ** +@@ -8700,7 +9087,7 @@ + ** checked out.</dd>)^ + ** + ** [[SQLITE_DBSTATUS_LOOKASIDE_HIT]] ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_HIT</dt> +-** <dd>This parameter returns the number malloc attempts that were ++** <dd>This parameter returns the number of malloc attempts that were + ** satisfied using lookaside memory. Only the high-water value is meaningful; + ** the current value is always zero.)^ + ** +@@ -8782,7 +9169,7 @@ + ** cache overflowing. Transactions are more efficient if they are written + ** to disk all at once. When pages spill mid-transaction, that introduces + ** additional overhead. This parameter can be used help identify +-** inefficiencies that can be resolve by increasing the cache size. ++** inefficiencies that can be resolved by increasing the cache size. + ** </dd> + ** + ** [[SQLITE_DBSTATUS_DEFERRED_FKS]] ^(<dt>SQLITE_DBSTATUS_DEFERRED_FKS</dt> +@@ -8871,7 +9258,7 @@ + ** + ** [[SQLITE_STMTSTATUS_REPREPARE]] <dt>SQLITE_STMTSTATUS_REPREPARE</dt> + ** <dd>^This is the number of times that the prepare statement has been +-** automatically regenerated due to schema changes or change to ++** automatically regenerated due to schema changes or changes to + ** [bound parameters] that might affect the query plan. + ** + ** [[SQLITE_STMTSTATUS_RUN]] <dt>SQLITE_STMTSTATUS_RUN</dt> +@@ -9042,7 +9429,7 @@ + ** + ** ^(SQLite will normally invoke xFetch() with a createFlag of 0 or 1. SQLite + ** will only use a createFlag of 2 after a prior call with a createFlag of 1 +-** failed.)^ In between the to xFetch() calls, SQLite may ++** failed.)^ In between the xFetch() calls, SQLite may + ** attempt to unpin one or more cache pages by spilling the content of + ** pinned pages to disk and synching the operating system disk cache. + ** +@@ -9360,7 +9747,7 @@ + ** the first argument to register for a callback that will be invoked + ** when the blocking connections current transaction is concluded. ^The + ** callback is invoked from within the [sqlite3_step] or [sqlite3_close] +-** call that concludes the blocking connections transaction. ++** call that concludes the blocking connection's transaction. + ** + ** ^(If sqlite3_unlock_notify() is called in a multi-threaded application, + ** there is a chance that the blocking connection will have already +@@ -9398,7 +9785,7 @@ + ** an unlock-notify callback is a pointer to an array of void* pointers, + ** and the second is the number of entries in the array. + ** +-** When a blocking connections transaction is concluded, there may be ++** When a blocking connection's transaction is concluded, there may be + ** more than one blocked connection that has registered for an unlock-notify + ** callback. ^If two or more such blocked connections have specified the + ** same callback function, then instead of invoking the callback function +@@ -9746,14 +10133,20 @@ + ** If this interface is invoked outside the context of an xConnect or + ** xCreate virtual table method then the behavior is undefined. + ** +-** At present, there is only one option that may be configured using +-** this function. (See [SQLITE_VTAB_CONSTRAINT_SUPPORT].) Further options +-** may be added in the future. ++** In the call sqlite3_vtab_config(D,C,...) the D parameter is the ++** [database connection] in which the virtual table is being created and ++** which is passed in as the first argument to the [xConnect] or [xCreate] ++** method that is invoking sqlite3_vtab_config(). The C parameter is one ++** of the [virtual table configuration options]. The presence and meaning ++** of parameters after C depend on which [virtual table configuration option] ++** is used. + */ + SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); + + /* + ** CAPI3REF: Virtual Table Configuration Options ++** KEYWORDS: {virtual table configuration options} ++** KEYWORDS: {virtual table configuration option} + ** + ** These macros define the various options to the + ** [sqlite3_vtab_config()] interface that [virtual table] implementations +@@ -9761,7 +10154,7 @@ + ** + ** <dl> + ** [[SQLITE_VTAB_CONSTRAINT_SUPPORT]] +-** <dt>SQLITE_VTAB_CONSTRAINT_SUPPORT ++** <dt>SQLITE_VTAB_CONSTRAINT_SUPPORT</dt> + ** <dd>Calls of the form + ** [sqlite3_vtab_config](db,SQLITE_VTAB_CONSTRAINT_SUPPORT,X) are supported, + ** where X is an integer. If X is zero, then the [virtual table] whose +@@ -9790,9 +10183,31 @@ + ** return SQLITE_OK. Or, if this is not possible, it may return + ** SQLITE_CONSTRAINT, in which case SQLite falls back to OR ABORT + ** constraint handling. ++** </dd> ++** ++** [[SQLITE_VTAB_DIRECTONLY]]<dt>SQLITE_VTAB_DIRECTONLY</dt> ++** <dd>Calls of the form ++** [sqlite3_vtab_config](db,SQLITE_VTAB_DIRECTONLY) from within the ++** the [xConnect] or [xCreate] methods of a [virtual table] implmentation ++** prohibits that virtual table from being used from within triggers and ++** views. ++** </dd> ++** ++** [[SQLITE_VTAB_INNOCUOUS]]<dt>SQLITE_VTAB_INNOCUOUS</dt> ++** <dd>Calls of the form ++** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the ++** the [xConnect] or [xCreate] methods of a [virtual table] implmentation ++** identify that virtual table as being safe to use from within triggers ++** and views. Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the ++** virtual table can do no serious harm even if it is controlled by a ++** malicious hacker. Developers should avoid setting the SQLITE_VTAB_INNOCUOUS ++** flag unless absolutely necessary. ++** </dd> + ** </dl> + */ + #define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 ++#define SQLITE_VTAB_INNOCUOUS 2 ++#define SQLITE_VTAB_DIRECTONLY 3 + + /* + ** CAPI3REF: Determine The Virtual Table Conflict Policy +@@ -9872,15 +10287,15 @@ + ** + ** <dl> + ** [[SQLITE_SCANSTAT_NLOOP]] <dt>SQLITE_SCANSTAT_NLOOP</dt> +-** <dd>^The [sqlite3_int64] variable pointed to by the T parameter will be ++** <dd>^The [sqlite3_int64] variable pointed to by the V parameter will be + ** set to the total number of times that the X-th loop has run.</dd> + ** + ** [[SQLITE_SCANSTAT_NVISIT]] <dt>SQLITE_SCANSTAT_NVISIT</dt> +-** <dd>^The [sqlite3_int64] variable pointed to by the T parameter will be set ++** <dd>^The [sqlite3_int64] variable pointed to by the V parameter will be set + ** to the total number of rows examined by all iterations of the X-th loop.</dd> + ** + ** [[SQLITE_SCANSTAT_EST]] <dt>SQLITE_SCANSTAT_EST</dt> +-** <dd>^The "double" variable pointed to by the T parameter will be set to the ++** <dd>^The "double" variable pointed to by the V parameter will be set to the + ** query planner's estimate for the average number of rows output from each + ** iteration of the X-th loop. If the query planner's estimates was accurate, + ** then this value will approximate the quotient NVISIT/NLOOP and the +@@ -9888,17 +10303,17 @@ + ** be the NLOOP value for the current loop. + ** + ** [[SQLITE_SCANSTAT_NAME]] <dt>SQLITE_SCANSTAT_NAME</dt> +-** <dd>^The "const char *" variable pointed to by the T parameter will be set ++** <dd>^The "const char *" variable pointed to by the V parameter will be set + ** to a zero-terminated UTF-8 string containing the name of the index or table + ** used for the X-th loop. + ** + ** [[SQLITE_SCANSTAT_EXPLAIN]] <dt>SQLITE_SCANSTAT_EXPLAIN</dt> +-** <dd>^The "const char *" variable pointed to by the T parameter will be set ++** <dd>^The "const char *" variable pointed to by the V parameter will be set + ** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] + ** description for the X-th loop. + ** + ** [[SQLITE_SCANSTAT_SELECTID]] <dt>SQLITE_SCANSTAT_SELECT</dt> +-** <dd>^The "int" variable pointed to by the T parameter will be set to the ++** <dd>^The "int" variable pointed to by the V parameter will be set to the + ** "select-id" for the X-th loop. The select-id identifies which query or + ** subquery the loop is part of. The main query has a select-id of zero. + ** The select-id is the same value as is output in the first column +@@ -10753,7 +11168,7 @@ + ** The second argument (xFilter) is the "filter callback". For changes to rows + ** in tables that are not attached to the Session object, the filter is called + ** to determine whether changes to the table's rows should be tracked or not. +-** If xFilter returns 0, changes is not tracked. Note that once a table is ++** If xFilter returns 0, changes are not tracked. Note that once a table is + ** attached, xFilter will not be called again. + */ + SQLITE_API void sqlite3session_table_filter( +@@ -10927,7 +11342,7 @@ + ** It an error if database zFrom does not exist or does not contain the + ** required compatible table. + ** +-** If the operation successful, SQLITE_OK is returned. Otherwise, an SQLite ++** If the operation is successful, SQLITE_OK is returned. Otherwise, an SQLite + ** error code. In this case, if argument pzErrMsg is not NULL, *pzErrMsg + ** may be set to point to a buffer containing an English language error + ** message. It is the responsibility of the caller to free this buffer using +@@ -11064,7 +11479,7 @@ + ** CAPI3REF: Advance A Changeset Iterator + ** METHOD: sqlite3_changeset_iter + ** +-** This function may only be used with iterators created by function ++** This function may only be used with iterators created by the function + ** [sqlite3changeset_start()]. If it is called on an iterator passed to + ** a conflict-handler callback by [sqlite3changeset_apply()], SQLITE_MISUSE + ** is returned and the call has no effect. +@@ -11480,8 +11895,8 @@ + ** case, this function fails with SQLITE_SCHEMA. If the input changeset + ** appears to be corrupt and the corruption is detected, SQLITE_CORRUPT is + ** returned. Or, if an out-of-memory condition occurs during processing, this +-** function returns SQLITE_NOMEM. In all cases, if an error occurs the +-** final contents of the changegroup is undefined. ++** function returns SQLITE_NOMEM. In all cases, if an error occurs the state ++** of the final contents of the changegroup is undefined. + ** + ** If no error occurs, SQLITE_OK is returned. + */ +@@ -11656,7 +12071,7 @@ + ** + ** It is safe to execute SQL statements, including those that write to the + ** table that the callback related to, from within the xConflict callback. +-** This can be used to further customize the applications conflict ++** This can be used to further customize the application's conflict + ** resolution strategy. + ** + ** All changes made by these functions are enclosed in a savepoint transaction. +@@ -11966,7 +12381,7 @@ + ** + ** Argument pIn must point to a buffer containing a changeset nIn bytes + ** in size. This function allocates and populates a buffer with a copy +-** of the changeset rebased rebased according to the configuration of the ++** of the changeset rebased according to the configuration of the + ** rebaser object passed as the first argument. If successful, (*ppOut) + ** is set to point to the new buffer containing the rebased changeset and + ** (*pnOut) to its size in bytes and SQLITE_OK returned. It is the +@@ -12374,7 +12789,7 @@ + ** + ** xSetAuxdata(pFts5, pAux, xDelete) + ** +-** Save the pointer passed as the second argument as the extension functions ++** Save the pointer passed as the second argument as the extension function's + ** "auxiliary data". The pointer may then be retrieved by the current or any + ** future invocation of the same fts5 extension function made as part of + ** the same MATCH query using the xGetAuxdata() API. +@@ -12616,8 +13031,8 @@ + ** + ** There are several ways to approach this in FTS5: + ** +-** <ol><li> By mapping all synonyms to a single token. In this case, the +-** In the above example, this means that the tokenizer returns the ++** <ol><li> By mapping all synonyms to a single token. In this case, using ++** the above example, this means that the tokenizer returns the + ** same token for inputs "first" and "1st". Say that token is in + ** fact "first", so that when the user inserts the document "I won + ** 1st place" entries are added to the index for tokens "i", "won", +@@ -12938,9 +13353,12 @@ + + /* + ** The maximum value of a ?nnn wildcard that the parser will accept. ++** If the value exceeds 32767 then extra space is required for the Expr ++** structure. But otherwise, we believe that the number can be as large ++** as a signed 32-bit integer can hold. + */ + #ifndef SQLITE_MAX_VARIABLE_NUMBER +-# define SQLITE_MAX_VARIABLE_NUMBER 999 ++# define SQLITE_MAX_VARIABLE_NUMBER 32766 + #endif + + /* Maximum page size. The upper bound on this value is 65536. This a limit +@@ -13029,6 +13447,21 @@ + #pragma warn -spa /* Suspicious pointer arithmetic */ + #endif + ++/* ++** WAL mode depends on atomic aligned 32-bit loads and stores in a few ++** places. The following macros try to make this explicit. ++*/ ++#ifndef __has_feature ++# define __has_feature(x) 0 /* compatibility with non-clang compilers */ ++#endif ++#if GCC_VERSION>=4007000 || __has_feature(c_atomic) ++# define AtomicLoad(PTR) __atomic_load_n((PTR),__ATOMIC_RELAXED) ++# define AtomicStore(PTR,VAL) __atomic_store_n((PTR),(VAL),__ATOMIC_RELAXED) ++#else ++# define AtomicLoad(PTR) (*(PTR)) ++# define AtomicStore(PTR,VAL) (*(PTR) = (VAL)) ++#endif ++ + /* + ** Include standard header files as necessary + */ +@@ -13055,15 +13488,15 @@ + ** So we have to define the macros in different ways depending on the + ** compiler. + */ +-#if defined(__PTRDIFF_TYPE__) /* This case should work for GCC */ ++#if defined(HAVE_STDINT_H) /* Use this case if we have ANSI headers */ ++# define SQLITE_INT_TO_PTR(X) ((void*)(intptr_t)(X)) ++# define SQLITE_PTR_TO_INT(X) ((int)(intptr_t)(X)) ++#elif defined(__PTRDIFF_TYPE__) /* This case should work for GCC */ + # define SQLITE_INT_TO_PTR(X) ((void*)(__PTRDIFF_TYPE__)(X)) + # define SQLITE_PTR_TO_INT(X) ((int)(__PTRDIFF_TYPE__)(X)) + #elif !defined(__GNUC__) /* Works for compilers other than LLVM */ + # define SQLITE_INT_TO_PTR(X) ((void*)&((char*)0)[X]) + # define SQLITE_PTR_TO_INT(X) ((int)(((char*)X)-(char*)0)) +-#elif defined(HAVE_STDINT_H) /* Use this case if we have ANSI headers */ +-# define SQLITE_INT_TO_PTR(X) ((void*)(intptr_t)(X)) +-# define SQLITE_PTR_TO_INT(X) ((int)(intptr_t)(X)) + #else /* Generates a warning - but it always works */ + # define SQLITE_INT_TO_PTR(X) ((void*)(X)) + # define SQLITE_PTR_TO_INT(X) ((int)(X)) +@@ -13289,6 +13722,26 @@ + # define NEVER(X) (X) + #endif + ++/* ++** The harmless(X) macro indicates that expression X is usually false ++** but can be true without causing any problems, but we don't know of ++** any way to cause X to be true. ++** ++** In debugging and testing builds, this macro will abort if X is ever ++** true. In this way, developers are alerted to a possible test case ++** that causes X to be true. If a harmless macro ever fails, that is ++** an opportunity to change the macro into a testcase() and add a new ++** test case to the test suite. ++** ++** For normal production builds, harmless(X) is a no-op, since it does ++** not matter whether expression X is true or false. ++*/ ++#ifdef SQLITE_DEBUG ++# define harmless(X) assert(!(X)); ++#else ++# define harmless(X) ++#endif ++ + /* + ** Some conditionals are optimizations only. In other words, if the + ** conditionals are replaced with a constant 1 (true) or 0 (false) then +@@ -13553,100 +14006,105 @@ + #define TK_VIEW 79 + #define TK_VIRTUAL 80 + #define TK_WITH 81 +-#define TK_CURRENT 82 +-#define TK_FOLLOWING 83 +-#define TK_PARTITION 84 +-#define TK_PRECEDING 85 +-#define TK_RANGE 86 +-#define TK_UNBOUNDED 87 +-#define TK_EXCLUDE 88 +-#define TK_GROUPS 89 +-#define TK_OTHERS 90 +-#define TK_TIES 91 +-#define TK_REINDEX 92 +-#define TK_RENAME 93 +-#define TK_CTIME_KW 94 +-#define TK_ANY 95 +-#define TK_BITAND 96 +-#define TK_BITOR 97 +-#define TK_LSHIFT 98 +-#define TK_RSHIFT 99 +-#define TK_PLUS 100 +-#define TK_MINUS 101 +-#define TK_STAR 102 +-#define TK_SLASH 103 +-#define TK_REM 104 +-#define TK_CONCAT 105 +-#define TK_COLLATE 106 +-#define TK_BITNOT 107 +-#define TK_ON 108 +-#define TK_INDEXED 109 +-#define TK_STRING 110 +-#define TK_JOIN_KW 111 +-#define TK_CONSTRAINT 112 +-#define TK_DEFAULT 113 +-#define TK_NULL 114 +-#define TK_PRIMARY 115 +-#define TK_UNIQUE 116 +-#define TK_CHECK 117 +-#define TK_REFERENCES 118 +-#define TK_AUTOINCR 119 +-#define TK_INSERT 120 +-#define TK_DELETE 121 +-#define TK_UPDATE 122 +-#define TK_SET 123 +-#define TK_DEFERRABLE 124 +-#define TK_FOREIGN 125 +-#define TK_DROP 126 +-#define TK_UNION 127 +-#define TK_ALL 128 +-#define TK_EXCEPT 129 +-#define TK_INTERSECT 130 +-#define TK_SELECT 131 +-#define TK_VALUES 132 +-#define TK_DISTINCT 133 +-#define TK_DOT 134 +-#define TK_FROM 135 +-#define TK_JOIN 136 +-#define TK_USING 137 +-#define TK_ORDER 138 +-#define TK_GROUP 139 +-#define TK_HAVING 140 +-#define TK_LIMIT 141 +-#define TK_WHERE 142 +-#define TK_INTO 143 +-#define TK_NOTHING 144 +-#define TK_FLOAT 145 +-#define TK_BLOB 146 +-#define TK_INTEGER 147 +-#define TK_VARIABLE 148 +-#define TK_CASE 149 +-#define TK_WHEN 150 +-#define TK_THEN 151 +-#define TK_ELSE 152 +-#define TK_INDEX 153 +-#define TK_ALTER 154 +-#define TK_ADD 155 +-#define TK_WINDOW 156 +-#define TK_OVER 157 +-#define TK_FILTER 158 +-#define TK_TRUEFALSE 159 +-#define TK_ISNOT 160 +-#define TK_FUNCTION 161 +-#define TK_COLUMN 162 +-#define TK_AGG_FUNCTION 163 +-#define TK_AGG_COLUMN 164 +-#define TK_UMINUS 165 +-#define TK_UPLUS 166 +-#define TK_TRUTH 167 +-#define TK_REGISTER 168 +-#define TK_VECTOR 169 +-#define TK_SELECT_COLUMN 170 +-#define TK_IF_NULL_ROW 171 +-#define TK_ASTERISK 172 +-#define TK_SPAN 173 +-#define TK_SPACE 174 +-#define TK_ILLEGAL 175 ++#define TK_NULLS 82 ++#define TK_FIRST 83 ++#define TK_LAST 84 ++#define TK_CURRENT 85 ++#define TK_FOLLOWING 86 ++#define TK_PARTITION 87 ++#define TK_PRECEDING 88 ++#define TK_RANGE 89 ++#define TK_UNBOUNDED 90 ++#define TK_EXCLUDE 91 ++#define TK_GROUPS 92 ++#define TK_OTHERS 93 ++#define TK_TIES 94 ++#define TK_GENERATED 95 ++#define TK_ALWAYS 96 ++#define TK_REINDEX 97 ++#define TK_RENAME 98 ++#define TK_CTIME_KW 99 ++#define TK_ANY 100 ++#define TK_BITAND 101 ++#define TK_BITOR 102 ++#define TK_LSHIFT 103 ++#define TK_RSHIFT 104 ++#define TK_PLUS 105 ++#define TK_MINUS 106 ++#define TK_STAR 107 ++#define TK_SLASH 108 ++#define TK_REM 109 ++#define TK_CONCAT 110 ++#define TK_COLLATE 111 ++#define TK_BITNOT 112 ++#define TK_ON 113 ++#define TK_INDEXED 114 ++#define TK_STRING 115 ++#define TK_JOIN_KW 116 ++#define TK_CONSTRAINT 117 ++#define TK_DEFAULT 118 ++#define TK_NULL 119 ++#define TK_PRIMARY 120 ++#define TK_UNIQUE 121 ++#define TK_CHECK 122 ++#define TK_REFERENCES 123 ++#define TK_AUTOINCR 124 ++#define TK_INSERT 125 ++#define TK_DELETE 126 ++#define TK_UPDATE 127 ++#define TK_SET 128 ++#define TK_DEFERRABLE 129 ++#define TK_FOREIGN 130 ++#define TK_DROP 131 ++#define TK_UNION 132 ++#define TK_ALL 133 ++#define TK_EXCEPT 134 ++#define TK_INTERSECT 135 ++#define TK_SELECT 136 ++#define TK_VALUES 137 ++#define TK_DISTINCT 138 ++#define TK_DOT 139 ++#define TK_FROM 140 ++#define TK_JOIN 141 ++#define TK_USING 142 ++#define TK_ORDER 143 ++#define TK_GROUP 144 ++#define TK_HAVING 145 ++#define TK_LIMIT 146 ++#define TK_WHERE 147 ++#define TK_INTO 148 ++#define TK_NOTHING 149 ++#define TK_FLOAT 150 ++#define TK_BLOB 151 ++#define TK_INTEGER 152 ++#define TK_VARIABLE 153 ++#define TK_CASE 154 ++#define TK_WHEN 155 ++#define TK_THEN 156 ++#define TK_ELSE 157 ++#define TK_INDEX 158 ++#define TK_ALTER 159 ++#define TK_ADD 160 ++#define TK_WINDOW 161 ++#define TK_OVER 162 ++#define TK_FILTER 163 ++#define TK_COLUMN 164 ++#define TK_AGG_FUNCTION 165 ++#define TK_AGG_COLUMN 166 ++#define TK_TRUEFALSE 167 ++#define TK_ISNOT 168 ++#define TK_FUNCTION 169 ++#define TK_UMINUS 170 ++#define TK_UPLUS 171 ++#define TK_TRUTH 172 ++#define TK_REGISTER 173 ++#define TK_VECTOR 174 ++#define TK_SELECT_COLUMN 175 ++#define TK_IF_NULL_ROW 176 ++#define TK_ASTERISK 177 ++#define TK_SPAN 178 ++#define TK_SPACE 179 ++#define TK_ILLEGAL 180 + + /************** End of parse.h ***********************************************/ + /************** Continuing where we left off in sqliteInt.h ******************/ +@@ -13952,12 +14410,13 @@ + ** at run-time. + */ + #ifndef SQLITE_BYTEORDER +-# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ +- defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ +- defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ +- defined(__arm__) || defined(_M_ARM64) ++# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ ++ defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ ++ defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ ++ defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM64) + # define SQLITE_BYTEORDER 1234 +-# elif defined(sparc) || defined(__ppc__) ++# elif defined(sparc) || defined(__ppc__) || \ ++ defined(__ARMEB__) || defined(__AARCH64EB__) + # define SQLITE_BYTEORDER 4321 + # else + # define SQLITE_BYTEORDER 0 +@@ -14056,20 +14515,6 @@ + # define SQLITE_DEFAULT_MMAP_SIZE SQLITE_MAX_MMAP_SIZE + #endif + +-/* +-** Only one of SQLITE_ENABLE_STAT3 or SQLITE_ENABLE_STAT4 can be defined. +-** Priority is given to SQLITE_ENABLE_STAT4. If either are defined, also +-** define SQLITE_ENABLE_STAT3_OR_STAT4 +-*/ +-#ifdef SQLITE_ENABLE_STAT4 +-# undef SQLITE_ENABLE_STAT3 +-# define SQLITE_ENABLE_STAT3_OR_STAT4 1 +-#elif SQLITE_ENABLE_STAT3 +-# define SQLITE_ENABLE_STAT3_OR_STAT4 1 +-#elif SQLITE_ENABLE_STAT3_OR_STAT4 +-# undef SQLITE_ENABLE_STAT3_OR_STAT4 +-#endif +- + /* + ** SELECTTRACE_ENABLED will be either 1 or 0 depending on whether or not + ** the Select query generator tracing logic is turned on. +@@ -14094,7 +14539,6 @@ + int (*xBusyHandler)(void *,int); /* The busy callback */ + void *pBusyArg; /* First arg to busy callback */ + int nBusy; /* Incremented with each busy call */ +- u8 bExtraFileArg; /* Include sqlite3_file as callback arg */ + }; + + /* +@@ -14257,6 +14701,7 @@ + ** A bit in a Bitmask + */ + #define MASKBIT(n) (((Bitmask)1)<<(n)) ++#define MASKBIT64(n) (((u64)1)<<(n)) + #define MASKBIT32(n) (((unsigned int)1)<<(n)) + #define ALLBITS ((Bitmask)-1) + +@@ -14351,7 +14796,7 @@ + SQLITE_PRIVATE int sqlite3BtreeMaxPageCount(Btree*,int); + SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree*); + SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree*,int); +-SQLITE_PRIVATE int sqlite3BtreeGetOptimalReserve(Btree*); ++SQLITE_PRIVATE int sqlite3BtreeGetRequestedReserve(Btree*); + SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p); + SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *, int); + SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *); +@@ -14583,6 +15028,8 @@ + SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor*); + SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor*, int flags); + SQLITE_PRIVATE i64 sqlite3BtreeIntegerKey(BtCursor*); ++SQLITE_PRIVATE void sqlite3BtreeCursorPin(BtCursor*); ++SQLITE_PRIVATE void sqlite3BtreeCursorUnpin(BtCursor*); + #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC + SQLITE_PRIVATE i64 sqlite3BtreeOffset(BtCursor*); + #endif +@@ -14591,7 +15038,7 @@ + SQLITE_PRIVATE u32 sqlite3BtreePayloadSize(BtCursor*); + SQLITE_PRIVATE sqlite3_int64 sqlite3BtreeMaxRecordSize(BtCursor*); + +-SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*); ++SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck(sqlite3*,Btree*,int*aRoot,int nRoot,int,int*); + SQLITE_PRIVATE struct Pager *sqlite3BtreePager(Btree*); + SQLITE_PRIVATE i64 sqlite3BtreeRowCountEst(BtCursor*); + +@@ -14611,9 +15058,7 @@ + #endif + SQLITE_PRIVATE int sqlite3BtreeCursorIsValidNN(BtCursor*); + +-#ifndef SQLITE_OMIT_BTREECOUNT +-SQLITE_PRIVATE int sqlite3BtreeCount(BtCursor *, i64 *); +-#endif ++SQLITE_PRIVATE int sqlite3BtreeCount(sqlite3*, BtCursor*, i64*); + + #ifdef SQLITE_TEST + SQLITE_PRIVATE int sqlite3BtreeCursorInfo(BtCursor*, int*, int); +@@ -14869,30 +15314,30 @@ + #define OP_SeekLE 23 /* jump, synopsis: key=r[P3@P4] */ + #define OP_SeekGE 24 /* jump, synopsis: key=r[P3@P4] */ + #define OP_SeekGT 25 /* jump, synopsis: key=r[P3@P4] */ +-#define OP_IfNoHope 26 /* jump, synopsis: key=r[P3@P4] */ +-#define OP_NoConflict 27 /* jump, synopsis: key=r[P3@P4] */ +-#define OP_NotFound 28 /* jump, synopsis: key=r[P3@P4] */ +-#define OP_Found 29 /* jump, synopsis: key=r[P3@P4] */ +-#define OP_SeekRowid 30 /* jump, synopsis: intkey=r[P3] */ +-#define OP_NotExists 31 /* jump, synopsis: intkey=r[P3] */ +-#define OP_Last 32 /* jump */ +-#define OP_IfSmaller 33 /* jump */ +-#define OP_SorterSort 34 /* jump */ +-#define OP_Sort 35 /* jump */ +-#define OP_Rewind 36 /* jump */ +-#define OP_IdxLE 37 /* jump, synopsis: key=r[P3@P4] */ +-#define OP_IdxGT 38 /* jump, synopsis: key=r[P3@P4] */ +-#define OP_IdxLT 39 /* jump, synopsis: key=r[P3@P4] */ +-#define OP_IdxGE 40 /* jump, synopsis: key=r[P3@P4] */ +-#define OP_RowSetRead 41 /* jump, synopsis: r[P3]=rowset(P1) */ +-#define OP_RowSetTest 42 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ ++#define OP_IfNotOpen 26 /* jump, synopsis: if( !csr[P1] ) goto P2 */ ++#define OP_IfNoHope 27 /* jump, synopsis: key=r[P3@P4] */ ++#define OP_NoConflict 28 /* jump, synopsis: key=r[P3@P4] */ ++#define OP_NotFound 29 /* jump, synopsis: key=r[P3@P4] */ ++#define OP_Found 30 /* jump, synopsis: key=r[P3@P4] */ ++#define OP_SeekRowid 31 /* jump, synopsis: intkey=r[P3] */ ++#define OP_NotExists 32 /* jump, synopsis: intkey=r[P3] */ ++#define OP_Last 33 /* jump */ ++#define OP_IfSmaller 34 /* jump */ ++#define OP_SorterSort 35 /* jump */ ++#define OP_Sort 36 /* jump */ ++#define OP_Rewind 37 /* jump */ ++#define OP_IdxLE 38 /* jump, synopsis: key=r[P3@P4] */ ++#define OP_IdxGT 39 /* jump, synopsis: key=r[P3@P4] */ ++#define OP_IdxLT 40 /* jump, synopsis: key=r[P3@P4] */ ++#define OP_IdxGE 41 /* jump, synopsis: key=r[P3@P4] */ ++#define OP_RowSetRead 42 /* jump, synopsis: r[P3]=rowset(P1) */ + #define OP_Or 43 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ + #define OP_And 44 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ +-#define OP_Program 45 /* jump */ +-#define OP_FkIfZero 46 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ +-#define OP_IfPos 47 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +-#define OP_IfNotZero 48 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ +-#define OP_DecrJumpZero 49 /* jump, synopsis: if (--r[P1])==0 goto P2 */ ++#define OP_RowSetTest 45 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ ++#define OP_Program 46 /* jump */ ++#define OP_FkIfZero 47 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ ++#define OP_IfPos 48 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ ++#define OP_IfNotZero 49 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ + #define OP_IsNull 50 /* jump, same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ + #define OP_NotNull 51 /* jump, same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ + #define OP_Ne 52 /* jump, same as TK_NE, synopsis: IF r[P3]!=r[P1] */ +@@ -14902,83 +15347,83 @@ + #define OP_Lt 56 /* jump, same as TK_LT, synopsis: IF r[P3]<r[P1] */ + #define OP_Ge 57 /* jump, same as TK_GE, synopsis: IF r[P3]>=r[P1] */ + #define OP_ElseNotEq 58 /* jump, same as TK_ESCAPE */ +-#define OP_IncrVacuum 59 /* jump */ +-#define OP_VNext 60 /* jump */ +-#define OP_Init 61 /* jump, synopsis: Start at P2 */ +-#define OP_PureFunc0 62 +-#define OP_Function0 63 /* synopsis: r[P3]=func(r[P2@P5]) */ +-#define OP_PureFunc 64 +-#define OP_Function 65 /* synopsis: r[P3]=func(r[P2@P5]) */ +-#define OP_Return 66 +-#define OP_EndCoroutine 67 +-#define OP_HaltIfNull 68 /* synopsis: if r[P3]=null halt */ +-#define OP_Halt 69 +-#define OP_Integer 70 /* synopsis: r[P2]=P1 */ +-#define OP_Int64 71 /* synopsis: r[P2]=P4 */ +-#define OP_String 72 /* synopsis: r[P2]='P4' (len=P1) */ +-#define OP_Null 73 /* synopsis: r[P2..P3]=NULL */ +-#define OP_SoftNull 74 /* synopsis: r[P1]=NULL */ +-#define OP_Blob 75 /* synopsis: r[P2]=P4 (len=P1) */ +-#define OP_Variable 76 /* synopsis: r[P2]=parameter(P1,P4) */ +-#define OP_Move 77 /* synopsis: r[P2@P3]=r[P1@P3] */ +-#define OP_Copy 78 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +-#define OP_SCopy 79 /* synopsis: r[P2]=r[P1] */ +-#define OP_IntCopy 80 /* synopsis: r[P2]=r[P1] */ +-#define OP_ResultRow 81 /* synopsis: output=r[P1@P2] */ +-#define OP_CollSeq 82 +-#define OP_AddImm 83 /* synopsis: r[P1]=r[P1]+P2 */ +-#define OP_RealAffinity 84 +-#define OP_Cast 85 /* synopsis: affinity(r[P1]) */ +-#define OP_Permutation 86 +-#define OP_Compare 87 /* synopsis: r[P1@P3] <-> r[P2@P3] */ +-#define OP_IsTrue 88 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ +-#define OP_Offset 89 /* synopsis: r[P3] = sqlite_offset(P1) */ +-#define OP_Column 90 /* synopsis: r[P3]=PX */ +-#define OP_Affinity 91 /* synopsis: affinity(r[P1@P2]) */ +-#define OP_MakeRecord 92 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ +-#define OP_Count 93 /* synopsis: r[P2]=count() */ +-#define OP_ReadCookie 94 +-#define OP_SetCookie 95 +-#define OP_BitAnd 96 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ +-#define OP_BitOr 97 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ +-#define OP_ShiftLeft 98 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<<r[P1] */ +-#define OP_ShiftRight 99 /* same as TK_RSHIFT, synopsis: r[P3]=r[P2]>>r[P1] */ +-#define OP_Add 100 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ +-#define OP_Subtract 101 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ +-#define OP_Multiply 102 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ +-#define OP_Divide 103 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ +-#define OP_Remainder 104 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ +-#define OP_Concat 105 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ +-#define OP_ReopenIdx 106 /* synopsis: root=P2 iDb=P3 */ +-#define OP_BitNot 107 /* same as TK_BITNOT, synopsis: r[P2]= ~r[P1] */ +-#define OP_OpenRead 108 /* synopsis: root=P2 iDb=P3 */ +-#define OP_OpenWrite 109 /* synopsis: root=P2 iDb=P3 */ +-#define OP_String8 110 /* same as TK_STRING, synopsis: r[P2]='P4' */ +-#define OP_OpenDup 111 +-#define OP_OpenAutoindex 112 /* synopsis: nColumn=P2 */ +-#define OP_OpenEphemeral 113 /* synopsis: nColumn=P2 */ +-#define OP_SorterOpen 114 +-#define OP_SequenceTest 115 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ +-#define OP_OpenPseudo 116 /* synopsis: P3 columns in r[P2] */ +-#define OP_Close 117 +-#define OP_ColumnsUsed 118 +-#define OP_SeekHit 119 /* synopsis: seekHit=P2 */ +-#define OP_Sequence 120 /* synopsis: r[P2]=cursor[P1].ctr++ */ +-#define OP_NewRowid 121 /* synopsis: r[P2]=rowid */ +-#define OP_Insert 122 /* synopsis: intkey=r[P3] data=r[P2] */ +-#define OP_Delete 123 +-#define OP_ResetCount 124 +-#define OP_SorterCompare 125 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ +-#define OP_SorterData 126 /* synopsis: r[P2]=data */ +-#define OP_RowData 127 /* synopsis: r[P2]=data */ +-#define OP_Rowid 128 /* synopsis: r[P2]=rowid */ +-#define OP_NullRow 129 +-#define OP_SeekEnd 130 ++#define OP_DecrJumpZero 59 /* jump, synopsis: if (--r[P1])==0 goto P2 */ ++#define OP_IncrVacuum 60 /* jump */ ++#define OP_VNext 61 /* jump */ ++#define OP_Init 62 /* jump, synopsis: Start at P2 */ ++#define OP_PureFunc 63 /* synopsis: r[P3]=func(r[P2@NP]) */ ++#define OP_Function 64 /* synopsis: r[P3]=func(r[P2@NP]) */ ++#define OP_Return 65 ++#define OP_EndCoroutine 66 ++#define OP_HaltIfNull 67 /* synopsis: if r[P3]=null halt */ ++#define OP_Halt 68 ++#define OP_Integer 69 /* synopsis: r[P2]=P1 */ ++#define OP_Int64 70 /* synopsis: r[P2]=P4 */ ++#define OP_String 71 /* synopsis: r[P2]='P4' (len=P1) */ ++#define OP_Null 72 /* synopsis: r[P2..P3]=NULL */ ++#define OP_SoftNull 73 /* synopsis: r[P1]=NULL */ ++#define OP_Blob 74 /* synopsis: r[P2]=P4 (len=P1) */ ++#define OP_Variable 75 /* synopsis: r[P2]=parameter(P1,P4) */ ++#define OP_Move 76 /* synopsis: r[P2@P3]=r[P1@P3] */ ++#define OP_Copy 77 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ ++#define OP_SCopy 78 /* synopsis: r[P2]=r[P1] */ ++#define OP_IntCopy 79 /* synopsis: r[P2]=r[P1] */ ++#define OP_ResultRow 80 /* synopsis: output=r[P1@P2] */ ++#define OP_CollSeq 81 ++#define OP_AddImm 82 /* synopsis: r[P1]=r[P1]+P2 */ ++#define OP_RealAffinity 83 ++#define OP_Cast 84 /* synopsis: affinity(r[P1]) */ ++#define OP_Permutation 85 ++#define OP_Compare 86 /* synopsis: r[P1@P3] <-> r[P2@P3] */ ++#define OP_IsTrue 87 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ ++#define OP_Offset 88 /* synopsis: r[P3] = sqlite_offset(P1) */ ++#define OP_Column 89 /* synopsis: r[P3]=PX */ ++#define OP_Affinity 90 /* synopsis: affinity(r[P1@P2]) */ ++#define OP_MakeRecord 91 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ ++#define OP_Count 92 /* synopsis: r[P2]=count() */ ++#define OP_ReadCookie 93 ++#define OP_SetCookie 94 ++#define OP_ReopenIdx 95 /* synopsis: root=P2 iDb=P3 */ ++#define OP_OpenRead 96 /* synopsis: root=P2 iDb=P3 */ ++#define OP_OpenWrite 97 /* synopsis: root=P2 iDb=P3 */ ++#define OP_OpenDup 98 ++#define OP_OpenAutoindex 99 /* synopsis: nColumn=P2 */ ++#define OP_OpenEphemeral 100 /* synopsis: nColumn=P2 */ ++#define OP_BitAnd 101 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ ++#define OP_BitOr 102 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ ++#define OP_ShiftLeft 103 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<<r[P1] */ ++#define OP_ShiftRight 104 /* same as TK_RSHIFT, synopsis: r[P3]=r[P2]>>r[P1] */ ++#define OP_Add 105 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ ++#define OP_Subtract 106 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ ++#define OP_Multiply 107 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ ++#define OP_Divide 108 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ ++#define OP_Remainder 109 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ ++#define OP_Concat 110 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ ++#define OP_SorterOpen 111 ++#define OP_BitNot 112 /* same as TK_BITNOT, synopsis: r[P2]= ~r[P1] */ ++#define OP_SequenceTest 113 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ ++#define OP_OpenPseudo 114 /* synopsis: P3 columns in r[P2] */ ++#define OP_String8 115 /* same as TK_STRING, synopsis: r[P2]='P4' */ ++#define OP_Close 116 ++#define OP_ColumnsUsed 117 ++#define OP_SeekHit 118 /* synopsis: seekHit=P2 */ ++#define OP_Sequence 119 /* synopsis: r[P2]=cursor[P1].ctr++ */ ++#define OP_NewRowid 120 /* synopsis: r[P2]=rowid */ ++#define OP_Insert 121 /* synopsis: intkey=r[P3] data=r[P2] */ ++#define OP_Delete 122 ++#define OP_ResetCount 123 ++#define OP_SorterCompare 124 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ ++#define OP_SorterData 125 /* synopsis: r[P2]=data */ ++#define OP_RowData 126 /* synopsis: r[P2]=data */ ++#define OP_Rowid 127 /* synopsis: r[P2]=rowid */ ++#define OP_NullRow 128 ++#define OP_SeekEnd 129 ++#define OP_IdxInsert 130 /* synopsis: key=r[P2] */ + #define OP_SorterInsert 131 /* synopsis: key=r[P2] */ +-#define OP_IdxInsert 132 /* synopsis: key=r[P2] */ +-#define OP_IdxDelete 133 /* synopsis: key=r[P2@P3] */ +-#define OP_DeferredSeek 134 /* synopsis: Move P3 to P1.rowid if needed */ +-#define OP_IdxRowid 135 /* synopsis: r[P2]=rowid */ ++#define OP_IdxDelete 132 /* synopsis: key=r[P2@P3] */ ++#define OP_DeferredSeek 133 /* synopsis: Move P3 to P1.rowid if needed */ ++#define OP_IdxRowid 134 /* synopsis: r[P2]=rowid */ ++#define OP_FinishSeek 135 + #define OP_Destroy 136 + #define OP_Clear 137 + #define OP_ResetSorter 138 +@@ -14988,12 +15433,12 @@ + #define OP_LoadAnalysis 142 + #define OP_DropTable 143 + #define OP_DropIndex 144 +-#define OP_Real 145 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ +-#define OP_DropTrigger 146 +-#define OP_IntegrityCk 147 +-#define OP_RowSetAdd 148 /* synopsis: rowset(P1)=r[P2] */ +-#define OP_Param 149 +-#define OP_FkCounter 150 /* synopsis: fkctr[P1]+=P2 */ ++#define OP_DropTrigger 145 ++#define OP_IntegrityCk 146 ++#define OP_RowSetAdd 147 /* synopsis: rowset(P1)=r[P2] */ ++#define OP_Param 148 ++#define OP_FkCounter 149 /* synopsis: fkctr[P1]+=P2 */ ++#define OP_Real 150 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ + #define OP_MemMax 151 /* synopsis: r[P1]=max(r[P1],r[P2]) */ + #define OP_OffsetLimit 152 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ + #define OP_AggInverse 153 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ +@@ -15002,20 +15447,23 @@ + #define OP_AggValue 156 /* synopsis: r[P3]=value N=P2 */ + #define OP_AggFinal 157 /* synopsis: accum=r[P1] N=P2 */ + #define OP_Expire 158 +-#define OP_TableLock 159 /* synopsis: iDb=P1 root=P2 write=P3 */ +-#define OP_VBegin 160 +-#define OP_VCreate 161 +-#define OP_VDestroy 162 +-#define OP_VOpen 163 +-#define OP_VColumn 164 /* synopsis: r[P3]=vcolumn(P2) */ +-#define OP_VRename 165 +-#define OP_Pagecount 166 +-#define OP_MaxPgcnt 167 +-#define OP_Trace 168 +-#define OP_CursorHint 169 +-#define OP_Noop 170 +-#define OP_Explain 171 +-#define OP_Abortable 172 ++#define OP_CursorLock 159 ++#define OP_CursorUnlock 160 ++#define OP_TableLock 161 /* synopsis: iDb=P1 root=P2 write=P3 */ ++#define OP_VBegin 162 ++#define OP_VCreate 163 ++#define OP_VDestroy 164 ++#define OP_VOpen 165 ++#define OP_VColumn 166 /* synopsis: r[P3]=vcolumn(P2) */ ++#define OP_VRename 167 ++#define OP_Pagecount 168 ++#define OP_MaxPgcnt 169 ++#define OP_Trace 170 ++#define OP_CursorHint 171 ++#define OP_ReleaseReg 172 /* synopsis: release r[P1@P2] mask P3 */ ++#define OP_Noop 173 ++#define OP_Explain 174 ++#define OP_Abortable 175 + + /* Properties such as "out2" or "jump" that are specified in + ** comments following the "case" for each opcode in the vdbe.c +@@ -15031,25 +15479,26 @@ + /* 0 */ 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x00, 0x10,\ + /* 8 */ 0x00, 0x01, 0x00, 0x01, 0x01, 0x01, 0x03, 0x03,\ + /* 16 */ 0x01, 0x01, 0x03, 0x12, 0x03, 0x01, 0x09, 0x09,\ +-/* 24 */ 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09,\ +-/* 32 */ 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\ +-/* 40 */ 0x01, 0x23, 0x0b, 0x26, 0x26, 0x01, 0x01, 0x03,\ ++/* 24 */ 0x09, 0x09, 0x01, 0x09, 0x09, 0x09, 0x09, 0x09,\ ++/* 32 */ 0x09, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\ ++/* 40 */ 0x01, 0x01, 0x23, 0x26, 0x26, 0x0b, 0x01, 0x01,\ + /* 48 */ 0x03, 0x03, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ +-/* 56 */ 0x0b, 0x0b, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00,\ +-/* 64 */ 0x00, 0x00, 0x02, 0x02, 0x08, 0x00, 0x10, 0x10,\ +-/* 72 */ 0x10, 0x10, 0x00, 0x10, 0x10, 0x00, 0x00, 0x10,\ +-/* 80 */ 0x10, 0x00, 0x00, 0x02, 0x02, 0x02, 0x00, 0x00,\ +-/* 88 */ 0x12, 0x20, 0x00, 0x00, 0x00, 0x10, 0x10, 0x00,\ +-/* 96 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26,\ +-/* 104 */ 0x26, 0x26, 0x00, 0x12, 0x00, 0x00, 0x10, 0x00,\ +-/* 112 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +-/* 120 */ 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +-/* 128 */ 0x10, 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x10,\ ++/* 56 */ 0x0b, 0x0b, 0x01, 0x03, 0x01, 0x01, 0x01, 0x00,\ ++/* 64 */ 0x00, 0x02, 0x02, 0x08, 0x00, 0x10, 0x10, 0x10,\ ++/* 72 */ 0x10, 0x00, 0x10, 0x10, 0x00, 0x00, 0x10, 0x10,\ ++/* 80 */ 0x00, 0x00, 0x02, 0x02, 0x02, 0x00, 0x00, 0x12,\ ++/* 88 */ 0x20, 0x00, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00,\ ++/* 96 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, 0x26, 0x26,\ ++/* 104 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x00,\ ++/* 112 */ 0x12, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x10,\ ++/* 120 */ 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10,\ ++/* 128 */ 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x10, 0x00,\ + /* 136 */ 0x10, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00,\ +-/* 144 */ 0x00, 0x10, 0x00, 0x00, 0x06, 0x10, 0x00, 0x04,\ ++/* 144 */ 0x00, 0x00, 0x00, 0x06, 0x10, 0x00, 0x10, 0x04,\ + /* 152 */ 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +-/* 160 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x10,\ +-/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00,} ++/* 160 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ ++/* 168 */ 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ ++} + + /* The sqlite3P2Values() routine is able to run faster if it knows + ** the value of the largest JUMP opcode. The smaller the maximum +@@ -15057,7 +15506,7 @@ + ** generated this include file strives to group all JUMP opcodes + ** together near the beginning of the list. + */ +-#define SQLITE_MX_JUMP_OPCODE 61 /* Maximum JUMP opcode */ ++#define SQLITE_MX_JUMP_OPCODE 62 /* Maximum JUMP opcode */ + + /************** End of opcodes.h *********************************************/ + /************** Continuing where we left off in vdbe.h ***********************/ +@@ -15073,6 +15522,7 @@ + ** for a description of what each of these routines does. + */ + SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse*); ++SQLITE_PRIVATE Parse *sqlite3VdbeParser(Vdbe*); + SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe*,int); + SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe*,int,int); + SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int); +@@ -15083,6 +15533,7 @@ + SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); + SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8(Vdbe*,int,int,int,int,const u8*,int); + SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); ++SQLITE_PRIVATE int sqlite3VdbeAddFunctionCall(Parse*,int,int,int,int,const FuncDef*,int); + SQLITE_PRIVATE void sqlite3VdbeEndCoroutine(Vdbe*,int); + #if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) + SQLITE_PRIVATE void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N); +@@ -15116,14 +15567,20 @@ + # define sqlite3ExplainBreakpoint(A,B) /*no-op*/ + #endif + SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); +-SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8); +-SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1); +-SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2); +-SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3); ++SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe*, int addr, u8); ++SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); ++SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); ++SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3); + SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u16 P5); + SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr); ++SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe*, int addr); + SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe*, int addr); + SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op); ++#ifdef SQLITE_DEBUG ++SQLITE_PRIVATE void sqlite3VdbeReleaseRegisters(Parse*,int addr, int n, u32 mask, int); ++#else ++# define sqlite3VdbeReleaseRegisters(P,A,N,M,F) ++#endif + SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N); + SQLITE_PRIVATE void sqlite3VdbeAppendP4(Vdbe*, void *pP4, int p4type); + SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); +@@ -15172,11 +15629,13 @@ + typedef int (*RecordCompare)(int,const void*,UnpackedRecord*); + SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); + +-#ifndef SQLITE_OMIT_TRIGGER + SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); +-#endif ++SQLITE_PRIVATE int sqlite3VdbeHasSubProgram(Vdbe*); + + SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context*); ++#ifdef SQLITE_ENABLE_BYTECODE_VTAB ++SQLITE_PRIVATE int sqlite3VdbeBytecodeVtabInit(sqlite3*); ++#endif + + /* Use SQLITE_ENABLE_COMMENTS to enable generation of extra comments on + ** each VDBE opcode. +@@ -15413,9 +15872,6 @@ + /* Functions used to configure a Pager object. */ + SQLITE_PRIVATE void sqlite3PagerSetBusyHandler(Pager*, int(*)(void *), void *); + SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int); +-#ifdef SQLITE_HAS_CODEC +-SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager*,Pager*); +-#endif + SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager*, int); + SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int); + SQLITE_PRIVATE int sqlite3PagerSetSpillsize(Pager*, int); +@@ -15465,14 +15921,22 @@ + SQLITE_PRIVATE int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); + SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager, sqlite3*); + # ifdef SQLITE_ENABLE_SNAPSHOT +-SQLITE_PRIVATE int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot); +-SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot); ++SQLITE_PRIVATE int sqlite3PagerSnapshotGet(Pager*, sqlite3_snapshot **ppSnapshot); ++SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager*, sqlite3_snapshot *pSnapshot); + SQLITE_PRIVATE int sqlite3PagerSnapshotRecover(Pager *pPager); + SQLITE_PRIVATE int sqlite3PagerSnapshotCheck(Pager *pPager, sqlite3_snapshot *pSnapshot); + SQLITE_PRIVATE void sqlite3PagerSnapshotUnlock(Pager *pPager); + # endif + #endif + ++#if !defined(SQLITE_OMIT_WAL) && defined(SQLITE_ENABLE_SETLK_TIMEOUT) ++SQLITE_PRIVATE int sqlite3PagerWalWriteLock(Pager*, int); ++SQLITE_PRIVATE void sqlite3PagerWalDb(Pager*, sqlite3*); ++#else ++# define sqlite3PagerWalWriteLock(y,z) SQLITE_OK ++# define sqlite3PagerWalDb(x,y) ++#endif ++ + #ifdef SQLITE_DIRECT_OVERFLOW_READ + SQLITE_PRIVATE int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno); + #endif +@@ -15488,7 +15952,7 @@ + SQLITE_PRIVATE int sqlite3PagerRefcount(Pager*); + #endif + SQLITE_PRIVATE int sqlite3PagerMemUsed(Pager*); +-SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager*, int); ++SQLITE_PRIVATE const char *sqlite3PagerFilename(const Pager*, int); + SQLITE_PRIVATE sqlite3_vfs *sqlite3PagerVfs(Pager*); + SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager*); + SQLITE_PRIVATE sqlite3_file *sqlite3PagerJrnlFile(Pager*); +@@ -15498,21 +15962,12 @@ + SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *, int, int, int *); + SQLITE_PRIVATE void sqlite3PagerClearCache(Pager*); + SQLITE_PRIVATE int sqlite3SectorSize(sqlite3_file *); +-#ifdef SQLITE_ENABLE_SETLK_TIMEOUT +-SQLITE_PRIVATE void sqlite3PagerResetLockTimeout(Pager *pPager); +-#else +-# define sqlite3PagerResetLockTimeout(X) +-#endif + + /* Functions used to truncate the database file. */ + SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager*,Pgno); + + SQLITE_PRIVATE void sqlite3PagerRekey(DbPage*, Pgno, u16); + +-#if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) +-SQLITE_PRIVATE void *sqlite3PagerCodec(DbPage *); +-#endif +- + /* Functions to support testing and debugging. */ + #if !defined(NDEBUG) || defined(SQLITE_TEST) + SQLITE_PRIVATE Pgno sqlite3PagerPagenumber(DbPage*); +@@ -16076,6 +16531,7 @@ + #define MUTEX_LOGIC(X) + #else + #define MUTEX_LOGIC(X) X ++SQLITE_API int sqlite3_mutex_held(sqlite3_mutex*); + #endif /* defined(SQLITE_MUTEX_OMIT) */ + + /************** End of mutex.h ***********************************************/ +@@ -16179,7 +16635,6 @@ + */ + #define DB_SchemaLoaded 0x0001 /* The schema has been loaded */ + #define DB_UnresetViews 0x0002 /* Some views have defined column names */ +-#define DB_Empty 0x0004 /* The file is empty (length 0 bytes) */ + #define DB_ResetWanted 0x0008 /* Reset the schema when nSchemaLock==0 */ + + /* +@@ -16207,15 +16662,47 @@ + ** is shared by multiple database connections. Therefore, while parsing + ** schema information, the Lookaside.bEnabled flag is cleared so that + ** lookaside allocations are not used to construct the schema objects. ++** ++** New lookaside allocations are only allowed if bDisable==0. When ++** bDisable is greater than zero, sz is set to zero which effectively ++** disables lookaside without adding a new test for the bDisable flag ++** in a performance-critical path. sz should be set by to szTrue whenever ++** bDisable changes back to zero. ++** ++** Lookaside buffers are initially held on the pInit list. As they are ++** used and freed, they are added back to the pFree list. New allocations ++** come off of pFree first, then pInit as a fallback. This dual-list ++** allows use to compute a high-water mark - the maximum number of allocations ++** outstanding at any point in the past - by subtracting the number of ++** allocations on the pInit list from the total number of allocations. ++** ++** Enhancement on 2019-12-12: Two-size-lookaside ++** The default lookaside configuration is 100 slots of 1200 bytes each. ++** The larger slot sizes are important for performance, but they waste ++** a lot of space, as most lookaside allocations are less than 128 bytes. ++** The two-size-lookaside enhancement breaks up the lookaside allocation ++** into two pools: One of 128-byte slots and the other of the default size ++** (1200-byte) slots. Allocations are filled from the small-pool first, ++** failing over to the full-size pool if that does not work. Thus more ++** lookaside slots are available while also using less memory. ++** This enhancement can be omitted by compiling with ++** SQLITE_OMIT_TWOSIZE_LOOKASIDE. + */ + struct Lookaside { + u32 bDisable; /* Only operate the lookaside when zero */ + u16 sz; /* Size of each buffer in bytes */ ++ u16 szTrue; /* True value of sz, even if disabled */ + u8 bMalloced; /* True if pStart obtained from sqlite3_malloc() */ + u32 nSlot; /* Number of lookaside slots allocated */ + u32 anStat[3]; /* 0: hits. 1: size misses. 2: full misses */ + LookasideSlot *pInit; /* List of buffers not previously used */ + LookasideSlot *pFree; /* List of available buffers */ ++#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE ++ LookasideSlot *pSmallInit; /* List of small buffers not prediously used */ ++ LookasideSlot *pSmallFree; /* List of available small buffers */ ++ void *pMiddle; /* First byte past end of full-size buffers and ++ ** the first byte of LOOKASIDE_SMALL buffers */ ++#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + void *pStart; /* First byte of available memory space */ + void *pEnd; /* First byte past end of available space */ + }; +@@ -16223,6 +16710,17 @@ + LookasideSlot *pNext; /* Next buffer in the list of free buffers */ + }; + ++#define DisableLookaside db->lookaside.bDisable++;db->lookaside.sz=0 ++#define EnableLookaside db->lookaside.bDisable--;\ ++ db->lookaside.sz=db->lookaside.bDisable?0:db->lookaside.szTrue ++ ++/* Size of the smaller allocations in two-size lookside */ ++#ifdef SQLITE_OMIT_TWOSIZE_LOOKASIDE ++# define LOOKASIDE_SMALL 0 ++#else ++# define LOOKASIDE_SMALL 128 ++#endif ++ + /* + ** A hash table for built-in function definitions. (Application-defined + ** functions use a regular table table from hash.h.) +@@ -16294,7 +16792,7 @@ + struct sqlite3 { + sqlite3_vfs *pVfs; /* OS Interface */ + struct Vdbe *pVdbe; /* List of active virtual machines */ +- CollSeq *pDfltColl; /* The default collating sequence (BINARY) */ ++ CollSeq *pDfltColl; /* BINARY collseq for the database encoding */ + sqlite3_mutex *mutex; /* Connection mutex */ + Db *aDb; /* All backends */ + int nDb; /* Number of backends currently in use */ +@@ -16334,6 +16832,7 @@ + unsigned orphanTrigger : 1; /* Last statement is orphaned TEMP trigger */ + unsigned imposterTable : 1; /* Building an imposter table */ + unsigned reopenMemdb : 1; /* ATTACH is really a reopen using MemDB */ ++ char **azInit; /* "type", "name", and "tbl_name" columns */ + } init; + int nVdbeActive; /* Number of VDBEs currently running */ + int nVdbeRead; /* Number of active VDBEs that read or write */ +@@ -16396,6 +16895,7 @@ + BusyHandler busyHandler; /* Busy callback */ + Db aDbStatic[2]; /* Static space for the 2 default backends */ + Savepoint *pSavepoint; /* List of active savepoints */ ++ int nAnalysisLimit; /* Number of index rows to ANALYZE */ + int busyTimeout; /* Busy handler timeout, in msec */ + int nSavepoint; /* Number of non-transaction savepoints */ + int nStatement; /* Number of nested statement-transactions */ +@@ -16430,6 +16930,13 @@ + #define SCHEMA_ENC(db) ((db)->aDb[0].pSchema->enc) + #define ENC(db) ((db)->enc) + ++/* ++** A u64 constant where the lower 32 bits are all zeros. Only the ++** upper 32 bits are included in the argument. Necessary because some ++** C-compilers still do not accept LL integer literals. ++*/ ++#define HI(X) ((u64)(X)<<32) ++ + /* + ** Possible values for the sqlite3.flags. + ** +@@ -16445,9 +16952,8 @@ + #define SQLITE_CkptFullFSync 0x00000010 /* Use full fsync for checkpoint */ + #define SQLITE_CacheSpill 0x00000020 /* OK to spill pager cache */ + #define SQLITE_ShortColNames 0x00000040 /* Show short columns names */ +-#define SQLITE_CountRows 0x00000080 /* Count rows changed by INSERT, */ +- /* DELETE, or UPDATE and return */ +- /* the count using a callback. */ ++#define SQLITE_TrustedSchema 0x00000080 /* Allow unsafe functions and ++ ** vtabs in the schema definition */ + #define SQLITE_NullCallback 0x00000100 /* Invoke the callback once if the */ + /* result set is empty */ + #define SQLITE_IgnoreChecks 0x00000200 /* Do not enforce check constraints */ +@@ -16470,16 +16976,21 @@ + #define SQLITE_LegacyAlter 0x04000000 /* Legacy ALTER TABLE behaviour */ + #define SQLITE_NoSchemaError 0x08000000 /* Do not report schema parse errors*/ + #define SQLITE_Defensive 0x10000000 /* Input SQL is likely hostile */ ++#define SQLITE_DqsDDL 0x20000000 /* dbl-quoted strings allowed in DDL*/ ++#define SQLITE_DqsDML 0x40000000 /* dbl-quoted strings allowed in DML*/ ++#define SQLITE_EnableView 0x80000000 /* Enable the use of views */ ++#define SQLITE_CountRows HI(0x00001) /* Count rows changed by INSERT, */ ++ /* DELETE, or UPDATE and return */ ++ /* the count using a callback. */ + + /* Flags used only if debugging */ +-#define HI(X) ((u64)(X)<<32) + #ifdef SQLITE_DEBUG +-#define SQLITE_SqlTrace HI(0x0001) /* Debug print SQL as it executes */ +-#define SQLITE_VdbeListing HI(0x0002) /* Debug listings of VDBE progs */ +-#define SQLITE_VdbeTrace HI(0x0004) /* True to trace VDBE execution */ +-#define SQLITE_VdbeAddopTrace HI(0x0008) /* Trace sqlite3VdbeAddOp() calls */ +-#define SQLITE_VdbeEQP HI(0x0010) /* Debug EXPLAIN QUERY PLAN */ +-#define SQLITE_ParserTrace HI(0x0020) /* PRAGMA parser_trace=ON */ ++#define SQLITE_SqlTrace HI(0x0100000) /* Debug print SQL as it executes */ ++#define SQLITE_VdbeListing HI(0x0200000) /* Debug listings of VDBE progs */ ++#define SQLITE_VdbeTrace HI(0x0400000) /* True to trace VDBE execution */ ++#define SQLITE_VdbeAddopTrace HI(0x0800000) /* Trace sqlite3VdbeAddOp() calls */ ++#define SQLITE_VdbeEQP HI(0x1000000) /* Debug EXPLAIN QUERY PLAN */ ++#define SQLITE_ParserTrace HI(0x2000000) /* PRAGMA parser_trace=ON */ + #endif + + /* +@@ -16490,6 +17001,8 @@ + #define DBFLAG_Vacuum 0x0004 /* Currently in a VACUUM */ + #define DBFLAG_VacuumInto 0x0008 /* Currently running VACUUM INTO */ + #define DBFLAG_SchemaKnownOk 0x0010 /* Schema is known to be valid */ ++#define DBFLAG_InternalFunc 0x0020 /* Allow use of internal functions */ ++#define DBFLAG_EncodingFixed 0x0040 /* No longer possible to change enc. */ + + /* + ** Bits of the sqlite3.dbOptFlags field that are used by the +@@ -16507,8 +17020,8 @@ + #define SQLITE_OmitNoopJoin 0x0100 /* Omit unused tables in joins */ + #define SQLITE_CountOfView 0x0200 /* The count-of-view optimization */ + #define SQLITE_CursorHints 0x0400 /* Add OP_CursorHint opcodes */ +-#define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ +- /* TH3 expects the Stat34 ^^^^^^ value to be 0x0800. Don't change it */ ++#define SQLITE_Stat4 0x0800 /* Use STAT4 data */ ++ /* TH3 expects the Stat4 ^^^^^^ value to be 0x0800. Don't change it */ + #define SQLITE_PushDown 0x1000 /* The push-down optimization */ + #define SQLITE_SimplifyJoin 0x2000 /* Convert LEFT JOIN to JOIN */ + #define SQLITE_SkipScan 0x4000 /* Skip-scans */ +@@ -16596,6 +17109,8 @@ + ** SQLITE_FUNC_LENGTH == OPFLAG_LENGTHARG + ** SQLITE_FUNC_TYPEOF == OPFLAG_TYPEOFARG + ** SQLITE_FUNC_CONSTANT == SQLITE_DETERMINISTIC from the API ++** SQLITE_FUNC_DIRECT == SQLITE_DIRECTONLY from the API ++** SQLITE_FUNC_UNSAFE == SQLITE_INNOCUOUS + ** SQLITE_FUNC_ENCMASK depends on SQLITE_UTF* macros in the API + */ + #define SQLITE_FUNC_ENCMASK 0x0003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ +@@ -16606,16 +17121,29 @@ + #define SQLITE_FUNC_LENGTH 0x0040 /* Built-in length() function */ + #define SQLITE_FUNC_TYPEOF 0x0080 /* Built-in typeof() function */ + #define SQLITE_FUNC_COUNT 0x0100 /* Built-in count(*) aggregate */ +-#define SQLITE_FUNC_COALESCE 0x0200 /* Built-in coalesce() or ifnull() */ ++/* 0x0200 -- available for reuse */ + #define SQLITE_FUNC_UNLIKELY 0x0400 /* Built-in unlikely() function */ + #define SQLITE_FUNC_CONSTANT 0x0800 /* Constant inputs give a constant output */ + #define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ + #define SQLITE_FUNC_SLOCHNG 0x2000 /* "Slow Change". Value constant during a + ** single query - might change over time */ +-#define SQLITE_FUNC_AFFINITY 0x4000 /* Built-in affinity() function */ ++#define SQLITE_FUNC_TEST 0x4000 /* Built-in testing functions */ + #define SQLITE_FUNC_OFFSET 0x8000 /* Built-in sqlite_offset() function */ + #define SQLITE_FUNC_WINDOW 0x00010000 /* Built-in window-only function */ + #define SQLITE_FUNC_INTERNAL 0x00040000 /* For use by NestedParse() only */ ++#define SQLITE_FUNC_DIRECT 0x00080000 /* Not for use in TRIGGERs or VIEWs */ ++#define SQLITE_FUNC_SUBTYPE 0x00100000 /* Result likely to have sub-type */ ++#define SQLITE_FUNC_UNSAFE 0x00200000 /* Function has side effects */ ++#define SQLITE_FUNC_INLINE 0x00400000 /* Functions implemented in-line */ ++ ++/* Identifier numbers for each in-line function */ ++#define INLINEFUNC_coalesce 0 ++#define INLINEFUNC_implies_nonnull_row 1 ++#define INLINEFUNC_expr_implies_expr 2 ++#define INLINEFUNC_expr_compare 3 ++#define INLINEFUNC_affinity 4 ++#define INLINEFUNC_iif 5 ++#define INLINEFUNC_unlikely 99 /* Default case */ + + /* + ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are +@@ -16631,6 +17159,22 @@ + ** VFUNCTION(zName, nArg, iArg, bNC, xFunc) + ** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag. + ** ++** SFUNCTION(zName, nArg, iArg, bNC, xFunc) ++** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and ++** adds the SQLITE_DIRECTONLY flag. ++** ++** INLINE_FUNC(zName, nArg, iFuncId, mFlags) ++** zName is the name of a function that is implemented by in-line ++** byte code rather than by the usual callbacks. The iFuncId ++** parameter determines the function id. The mFlags parameter is ++** optional SQLITE_FUNC_ flags for this function. ++** ++** TEST_FUNC(zName, nArg, iFuncId, mFlags) ++** zName is the name of a test-only function implemented by in-line ++** byte code rather than by the usual callbacks. The iFuncId ++** parameter determines the function id. The mFlags parameter is ++** optional SQLITE_FUNC_ flags for this function. ++** + ** DFUNCTION(zName, nArg, iArg, bNC, xFunc) + ** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and + ** adds the SQLITE_FUNC_SLOCHNG flag. Used for date & time functions +@@ -16670,6 +17214,16 @@ + #define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } ++#define SFUNCTION(zName, nArg, iArg, bNC, xFunc) \ ++ {nArg, SQLITE_UTF8|SQLITE_DIRECTONLY|SQLITE_FUNC_UNSAFE, \ ++ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } ++#define INLINE_FUNC(zName, nArg, iArg, mFlags) \ ++ {nArg, SQLITE_UTF8|SQLITE_FUNC_INLINE|SQLITE_FUNC_CONSTANT|(mFlags), \ ++ SQLITE_INT_TO_PTR(iArg), 0, noopFunc, 0, 0, 0, #zName, {0} } ++#define TEST_FUNC(zName, nArg, iArg, mFlags) \ ++ {nArg, SQLITE_UTF8|SQLITE_FUNC_INTERNAL|SQLITE_FUNC_TEST| \ ++ SQLITE_FUNC_INLINE|SQLITE_FUNC_CONSTANT|(mFlags), \ ++ SQLITE_INT_TO_PTR(iArg), 0, noopFunc, 0, 0, 0, #zName, {0} } + #define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8, \ + 0, 0, xFunc, 0, 0, 0, #zName, {0} } +@@ -16685,12 +17239,6 @@ + #define LIKEFUNC(zName, nArg, arg, flags) \ + {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ + (void *)arg, 0, likeFunc, 0, 0, 0, #zName, {0} } +-#define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal, xValue) \ +- {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL), \ +- SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,xValue,0,#zName, {0}} +-#define AGGREGATE2(zName, nArg, arg, nc, xStep, xFinal, extraFlags) \ +- {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|extraFlags, \ +- SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,xFinal,0,#zName, {0}} + #define WAGGREGATE(zName, nArg, arg, nc, xStep, xFinal, xValue, xInverse, f) \ + {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|f, \ + SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,xValue,xInverse,#zName, {0}} +@@ -16729,32 +17277,53 @@ + struct Module { + const sqlite3_module *pModule; /* Callback pointers */ + const char *zName; /* Name passed to create_module() */ ++ int nRefModule; /* Number of pointers to this object */ + void *pAux; /* pAux passed to create_module() */ + void (*xDestroy)(void *); /* Module destructor function */ + Table *pEpoTab; /* Eponymous table for this module */ + }; + + /* +-** information about each column of an SQL table is held in an instance +-** of this structure. ++** Information about each column of an SQL table is held in an instance ++** of the Column structure, in the Table.aCol[] array. ++** ++** Definitions: ++** ++** "table column index" This is the index of the column in the ++** Table.aCol[] array, and also the index of ++** the column in the original CREATE TABLE stmt. ++** ++** "storage column index" This is the index of the column in the ++** record BLOB generated by the OP_MakeRecord ++** opcode. The storage column index is less than ++** or equal to the table column index. It is ++** equal if and only if there are no VIRTUAL ++** columns to the left. + */ + struct Column { + char *zName; /* Name of this column, \000, then the type */ +- Expr *pDflt; /* Default value of this column */ ++ Expr *pDflt; /* Default value or GENERATED ALWAYS AS value */ + char *zColl; /* Collating sequence. If NULL, use the default */ + u8 notNull; /* An OE_ code for handling a NOT NULL constraint */ + char affinity; /* One of the SQLITE_AFF_... values */ + u8 szEst; /* Estimated size of value in this column. sizeof(INT)==1 */ +- u8 colFlags; /* Boolean properties. See COLFLAG_ defines below */ ++ u8 hName; /* Column name hash for faster lookup */ ++ u16 colFlags; /* Boolean properties. See COLFLAG_ defines below */ + }; + + /* Allowed values for Column.colFlags: + */ +-#define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ +-#define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ +-#define COLFLAG_HASTYPE 0x0004 /* Type name follows column name */ +-#define COLFLAG_UNIQUE 0x0008 /* Column def contains "UNIQUE" or "PK" */ ++#define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ ++#define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ ++#define COLFLAG_HASTYPE 0x0004 /* Type name follows column name */ ++#define COLFLAG_UNIQUE 0x0008 /* Column def contains "UNIQUE" or "PK" */ + #define COLFLAG_SORTERREF 0x0010 /* Use sorter-refs with this column */ ++#define COLFLAG_VIRTUAL 0x0020 /* GENERATED ALWAYS AS ... VIRTUAL */ ++#define COLFLAG_STORED 0x0040 /* GENERATED ALWAYS AS ... STORED */ ++#define COLFLAG_NOTAVAIL 0x0080 /* STORED column not yet calculated */ ++#define COLFLAG_BUSY 0x0100 /* Blocks recursion on GENERATED columns */ ++#define COLFLAG_GENERATED 0x0060 /* Combo: _STORED, _VIRTUAL */ ++#define COLFLAG_NOINSERT 0x0062 /* Combo: _HIDDEN, _STORED, _VIRTUAL */ + + /* + ** A "Collating Sequence" is defined by an instance of the following +@@ -16794,11 +17363,12 @@ + ** Note also that the numeric types are grouped together so that testing + ** for a numeric type is a single comparison. And the BLOB type is first. + */ +-#define SQLITE_AFF_BLOB 'A' +-#define SQLITE_AFF_TEXT 'B' +-#define SQLITE_AFF_NUMERIC 'C' +-#define SQLITE_AFF_INTEGER 'D' +-#define SQLITE_AFF_REAL 'E' ++#define SQLITE_AFF_NONE 0x40 /* '@' */ ++#define SQLITE_AFF_BLOB 0x41 /* 'A' */ ++#define SQLITE_AFF_TEXT 0x42 /* 'B' */ ++#define SQLITE_AFF_NUMERIC 0x43 /* 'C' */ ++#define SQLITE_AFF_INTEGER 0x44 /* 'D' */ ++#define SQLITE_AFF_REAL 0x45 /* 'E' */ + + #define sqlite3IsNumericAffinity(X) ((X)>=SQLITE_AFF_NUMERIC) + +@@ -16871,10 +17441,17 @@ + sqlite3_vtab *pVtab; /* Pointer to vtab instance */ + int nRef; /* Number of pointers to this structure */ + u8 bConstraint; /* True if constraints are supported */ ++ u8 eVtabRisk; /* Riskiness of allowing hacker access */ + int iSavepoint; /* Depth of the SAVEPOINT stack */ + VTable *pNext; /* Next in linked list (see above) */ + }; + ++/* Allowed values for VTable.eVtabRisk ++*/ ++#define SQLITE_VTABRISK_Low 0 ++#define SQLITE_VTABRISK_Normal 1 ++#define SQLITE_VTABRISK_High 2 ++ + /* + ** The schema for each SQL table and view is represented in memory + ** by an instance of the following structure. +@@ -16893,6 +17470,7 @@ + u32 tabFlags; /* Mask of TF_* values */ + i16 iPKey; /* If not negative, use aCol[iPKey] as the rowid */ + i16 nCol; /* Number of columns in this table */ ++ i16 nNVCol; /* Number of columns that are not VIRTUAL */ + LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ + LogEst szTabRow; /* Estimated size of each table row in bytes */ + #ifdef SQLITE_ENABLE_COSTMULT +@@ -16919,20 +17497,28 @@ + ** followed by non-hidden columns. Example: "CREATE VIRTUAL TABLE x USING + ** vtab1(a HIDDEN, b);". Since "b" is a non-hidden column but "a" is hidden, + ** the TF_OOOHidden attribute would apply in this case. Such tables require +-** special handling during INSERT processing. ++** special handling during INSERT processing. The "OOO" means "Out Of Order". ++** ++** Constraints: ++** ++** TF_HasVirtual == COLFLAG_Virtual ++** TF_HasStored == COLFLAG_Stored + */ + #define TF_Readonly 0x0001 /* Read-only system table */ + #define TF_Ephemeral 0x0002 /* An ephemeral table */ + #define TF_HasPrimaryKey 0x0004 /* Table has a primary key */ + #define TF_Autoincrement 0x0008 /* Integer primary key is autoincrement */ + #define TF_HasStat1 0x0010 /* nRowLogEst set from sqlite_stat1 */ +-#define TF_WithoutRowid 0x0020 /* No rowid. PRIMARY KEY is the key */ +-#define TF_NoVisibleRowid 0x0040 /* No user-visible "rowid" column */ +-#define TF_OOOHidden 0x0080 /* Out-of-Order hidden columns */ ++#define TF_HasVirtual 0x0020 /* Has one or more VIRTUAL columns */ ++#define TF_HasStored 0x0040 /* Has one or more STORED columns */ ++#define TF_HasGenerated 0x0060 /* Combo: HasVirtual + HasStored */ ++#define TF_WithoutRowid 0x0080 /* No rowid. PRIMARY KEY is the key */ + #define TF_StatsUsed 0x0100 /* Query planner decisions affected by + ** Index.aiRowLogEst[] values */ +-#define TF_HasNotNull 0x0200 /* Contains NOT NULL constraints */ +-#define TF_Shadow 0x0400 /* True for a shadow table */ ++#define TF_NoVisibleRowid 0x0200 /* No user-visible "rowid" column */ ++#define TF_OOOHidden 0x0400 /* Out-of-Order hidden columns */ ++#define TF_HasNotNull 0x0800 /* Contains NOT NULL constraints */ ++#define TF_Shadow 0x1000 /* True for a shadow table */ + + /* + ** Test to see whether or not a table is a virtual table. This is +@@ -16941,8 +17527,11 @@ + */ + #ifndef SQLITE_OMIT_VIRTUALTABLE + # define IsVirtual(X) ((X)->nModuleArg) ++# define ExprIsVtab(X) \ ++ ((X)->op==TK_COLUMN && (X)->y.pTab!=0 && (X)->y.pTab->nModuleArg) + #else + # define IsVirtual(X) 0 ++# define ExprIsVtab(X) 0 + #endif + + /* +@@ -17066,10 +17655,16 @@ + u16 nKeyField; /* Number of key columns in the index */ + u16 nAllField; /* Total columns, including key plus others */ + sqlite3 *db; /* The database connection */ +- u8 *aSortOrder; /* Sort order for each column. */ ++ u8 *aSortFlags; /* Sort order for each column. */ + CollSeq *aColl[1]; /* Collating sequence for each term of the key */ + }; + ++/* ++** Allowed bit values for entries in the KeyInfo.aSortFlags[] array. ++*/ ++#define KEYINFO_ORDER_DESC 0x01 /* DESC sort order */ ++#define KEYINFO_ORDER_BIGNULL 0x02 /* NULL is larger than any other value */ ++ + /* + ** This object holds a record which has been parsed out into individual + ** fields, for the purposes of doing a comparison. +@@ -17176,7 +17771,9 @@ + unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ + unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ + unsigned bNoQuery:1; /* Do not use this index to optimize queries */ +-#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 ++ unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */ ++ unsigned bHasVCol:1; /* Index references one or more VIRTUAL columns */ ++#ifdef SQLITE_ENABLE_STAT4 + int nSample; /* Number of elements in aSample[] */ + int nSampleCol; /* Size of IndexSample.anEq[] and so on */ + tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ +@@ -17208,7 +17805,7 @@ + #define XN_EXPR (-2) /* Indexed column is an expression */ + + /* +-** Each sample stored in the sqlite_stat3 table is represented in memory ++** Each sample stored in the sqlite_stat4 table is represented in memory + ** using a structure of this type. See documentation at the top of the + ** analyze.c source file for additional information. + */ +@@ -17246,7 +17843,7 @@ + ** code for a SELECT that contains aggregate functions. + ** + ** If Expr.op==TK_AGG_COLUMN or TK_AGG_FUNCTION then Expr.pAggInfo is a +-** pointer to this structure. The Expr.iColumn field is the index in ++** pointer to this structure. The Expr.iAgg field is the index in + ** AggInfo.aCol[] or AggInfo.aFunc[] of information needed to generate + ** code for that node. + ** +@@ -17292,10 +17889,10 @@ + ** it uses less memory in the Expr object, which is a big memory user + ** in systems with lots of prepared statements. And few applications + ** need more than about 10 or 20 variables. But some extreme users want +-** to have prepared statements with over 32767 variables, and for them ++** to have prepared statements with over 32766 variables, and for them + ** the option is available (at compile-time). + */ +-#if SQLITE_MAX_VARIABLE_NUMBER<=32767 ++#if SQLITE_MAX_VARIABLE_NUMBER<32767 + typedef i16 ynVar; + #else + typedef int ynVar; +@@ -17366,7 +17963,14 @@ + */ + struct Expr { + u8 op; /* Operation performed by this node */ +- char affinity; /* The affinity of the column or 0 if not a column */ ++ char affExpr; /* affinity, or RAISE type */ ++ u8 op2; /* TK_REGISTER/TK_TRUTH: original value of Expr.op ++ ** TK_COLUMN: the value of p5 for OP_Column ++ ** TK_AGG_FUNCTION: nesting depth ++ ** TK_FUNCTION: NC_SelfRef flag if needs OP_PureFunc */ ++#ifdef SQLITE_DEBUG ++ u8 vvaFlags; /* Verification flags. */ ++#endif + u32 flags; /* Various flags. EP_* See below */ + union { + char *zToken; /* Token value. Zero terminated and dequoted */ +@@ -17397,20 +18001,19 @@ + ** TK_REGISTER: register number + ** TK_TRIGGER: 1 -> new, 0 -> old + ** EP_Unlikely: 134217728 times likelihood ++ ** TK_IN: ephemerial table holding RHS ++ ** TK_SELECT_COLUMN: Number of columns on the LHS + ** TK_SELECT: 1st register of result vector */ + ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. + ** TK_VARIABLE: variable number (always >= 1). + ** TK_SELECT_COLUMN: column of the result vector */ + i16 iAgg; /* Which entry in pAggInfo->aCol[] or ->aFunc[] */ + i16 iRightJoinTable; /* If EP_FromJoin, the right table of the join */ +- u8 op2; /* TK_REGISTER: original value of Expr.op +- ** TK_COLUMN: the value of p5 for OP_Column +- ** TK_AGG_FUNCTION: nesting depth */ + AggInfo *pAggInfo; /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */ + union { + Table *pTab; /* TK_COLUMN: Table containing column. Can be NULL + ** for a column of an index on an expression */ +- Window *pWin; /* TK_FUNCTION: Window definition for the func */ ++ Window *pWin; /* EP_WinFunc: Window/Filter defn for a function */ + struct { /* TK_IN, TK_SELECT, and TK_EXISTS */ + int iAddr; /* Subroutine entry address */ + int regReturn; /* Register used to hold return address */ +@@ -17425,34 +18028,38 @@ + ** EP_Agg == NC_HasAgg == SF_HasAgg + ** EP_Win == NC_HasWin + */ +-#define EP_FromJoin 0x000001 /* Originates in ON/USING clause of outer join */ +-#define EP_Distinct 0x000002 /* Aggregate function with DISTINCT keyword */ +-#define EP_HasFunc 0x000004 /* Contains one or more functions of any kind */ +-#define EP_FixedCol 0x000008 /* TK_Column with a known fixed value */ +-#define EP_Agg 0x000010 /* Contains one or more aggregate functions */ +-#define EP_VarSelect 0x000020 /* pSelect is correlated, not constant */ +-#define EP_DblQuoted 0x000040 /* token.z was originally in "..." */ +-#define EP_InfixFunc 0x000080 /* True for an infix function: LIKE, GLOB, etc */ +-#define EP_Collate 0x000100 /* Tree contains a TK_COLLATE operator */ +-#define EP_Generic 0x000200 /* Ignore COLLATE or affinity on this tree */ +-#define EP_IntValue 0x000400 /* Integer value contained in u.iValue */ +-#define EP_xIsSelect 0x000800 /* x.pSelect is valid (otherwise x.pList is) */ +-#define EP_Skip 0x001000 /* COLLATE, AS, or UNLIKELY */ +-#define EP_Reduced 0x002000 /* Expr struct EXPR_REDUCEDSIZE bytes only */ +-#define EP_TokenOnly 0x004000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */ +-#define EP_Win 0x008000 /* Contains window functions */ +-#define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ +-#define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ +-#define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ +-#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ +-#define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ +-#define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ +-#define EP_Alias 0x400000 /* Is an alias for a result set column */ +-#define EP_Leaf 0x800000 /* Expr.pLeft, .pRight, .u.pSelect all NULL */ +-#define EP_WinFunc 0x1000000 /* TK_FUNCTION with Expr.y.pWin set */ +-#define EP_Subrtn 0x2000000 /* Uses Expr.y.sub. TK_IN, _SELECT, or _EXISTS */ +-#define EP_Quoted 0x4000000 /* TK_ID was originally quoted */ +-#define EP_Static 0x8000000 /* Held in memory not obtained from malloc() */ ++#define EP_FromJoin 0x000001 /* Originates in ON/USING clause of outer join */ ++#define EP_Distinct 0x000002 /* Aggregate function with DISTINCT keyword */ ++#define EP_HasFunc 0x000004 /* Contains one or more functions of any kind */ ++#define EP_FixedCol 0x000008 /* TK_Column with a known fixed value */ ++#define EP_Agg 0x000010 /* Contains one or more aggregate functions */ ++#define EP_VarSelect 0x000020 /* pSelect is correlated, not constant */ ++#define EP_DblQuoted 0x000040 /* token.z was originally in "..." */ ++#define EP_InfixFunc 0x000080 /* True for an infix function: LIKE, GLOB, etc */ ++#define EP_Collate 0x000100 /* Tree contains a TK_COLLATE operator */ ++#define EP_Commuted 0x000200 /* Comparison operator has been commuted */ ++#define EP_IntValue 0x000400 /* Integer value contained in u.iValue */ ++#define EP_xIsSelect 0x000800 /* x.pSelect is valid (otherwise x.pList is) */ ++#define EP_Skip 0x001000 /* Operator does not contribute to affinity */ ++#define EP_Reduced 0x002000 /* Expr struct EXPR_REDUCEDSIZE bytes only */ ++#define EP_TokenOnly 0x004000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */ ++#define EP_Win 0x008000 /* Contains window functions */ ++#define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ ++ /* 0x020000 // available for reuse */ ++#define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ ++#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ ++#define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ ++#define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ ++#define EP_Alias 0x400000 /* Is an alias for a result set column */ ++#define EP_Leaf 0x800000 /* Expr.pLeft, .pRight, .u.pSelect all NULL */ ++#define EP_WinFunc 0x1000000 /* TK_FUNCTION with Expr.y.pWin set */ ++#define EP_Subrtn 0x2000000 /* Uses Expr.y.sub. TK_IN, _SELECT, or _EXISTS */ ++#define EP_Quoted 0x4000000 /* TK_ID was originally quoted */ ++#define EP_Static 0x8000000 /* Held in memory not obtained from malloc() */ ++#define EP_IsTrue 0x10000000 /* Always has boolean value of TRUE */ ++#define EP_IsFalse 0x20000000 /* Always has boolean value of FALSE */ ++#define EP_FromDDL 0x40000000 /* Originates from sqlite_master */ ++ /* 0x80000000 // Available */ + + /* + ** The EP_Propagate mask is a set of properties that automatically propagate +@@ -17468,15 +18075,27 @@ + #define ExprHasAllProperty(E,P) (((E)->flags&(P))==(P)) + #define ExprSetProperty(E,P) (E)->flags|=(P) + #define ExprClearProperty(E,P) (E)->flags&=~(P) ++#define ExprAlwaysTrue(E) (((E)->flags&(EP_FromJoin|EP_IsTrue))==EP_IsTrue) ++#define ExprAlwaysFalse(E) (((E)->flags&(EP_FromJoin|EP_IsFalse))==EP_IsFalse) ++ ++ ++/* Flags for use with Expr.vvaFlags ++*/ ++#define EP_NoReduce 0x01 /* Cannot EXPRDUP_REDUCE this Expr */ ++#define EP_Immutable 0x02 /* Do not change this Expr node */ + + /* The ExprSetVVAProperty() macro is used for Verification, Validation, + ** and Accreditation only. It works like ExprSetProperty() during VVA + ** processes but is a no-op for delivery. + */ + #ifdef SQLITE_DEBUG +-# define ExprSetVVAProperty(E,P) (E)->flags|=(P) ++# define ExprSetVVAProperty(E,P) (E)->vvaFlags|=(P) ++# define ExprHasVVAProperty(E,P) (((E)->vvaFlags&(P))!=0) ++# define ExprClearVVAProperties(E) (E)->vvaFlags = 0 + #else + # define ExprSetVVAProperty(E,P) ++# define ExprHasVVAProperty(E,P) 0 ++# define ExprClearVVAProperties(E) + #endif + + /* +@@ -17494,6 +18113,18 @@ + */ + #define EXPRDUP_REDUCE 0x0001 /* Used reduced-size Expr nodes */ + ++/* ++** True if the expression passed as an argument was a function with ++** an OVER() clause (a window function). ++*/ ++#ifdef SQLITE_OMIT_WINDOWFUNC ++# define IsWindowFunc(p) 0 ++#else ++# define IsWindowFunc(p) ( \ ++ ExprHasProperty((p), EP_WinFunc) && p->y.pWin->eFrmType!=TK_FILTER \ ++ ) ++#endif ++ + /* + ** A list of expressions. Each expression may optionally have a + ** name. An expr/name combination can be used in several ways, such +@@ -17502,25 +18133,31 @@ + ** also be used as the argument to a function, in which case the a.zName + ** field is not used. + ** +-** By default the Expr.zSpan field holds a human-readable description of +-** the expression that is used in the generation of error messages and +-** column labels. In this case, Expr.zSpan is typically the text of a +-** column expression as it exists in a SELECT statement. However, if +-** the bSpanIsTab flag is set, then zSpan is overloaded to mean the name +-** of the result column in the form: DATABASE.TABLE.COLUMN. This later +-** form is used for name resolution with nested FROM clauses. ++** In order to try to keep memory usage down, the Expr.a.zEName field ++** is used for multiple purposes: ++** ++** eEName Usage ++** ---------- ------------------------- ++** ENAME_NAME (1) the AS of result set column ++** (2) COLUMN= of an UPDATE ++** ++** ENAME_TAB DB.TABLE.NAME used to resolve names ++** of subqueries ++** ++** ENAME_SPAN Text of the original result set ++** expression. + */ + struct ExprList { + int nExpr; /* Number of expressions on the list */ + struct ExprList_item { /* For each expression in the list */ + Expr *pExpr; /* The parse tree for this expression */ +- char *zName; /* Token associated with this expression */ +- char *zSpan; /* Original text of the expression */ +- u8 sortOrder; /* 1 for DESC or 0 for ASC */ ++ char *zEName; /* Token associated with this expression */ ++ u8 sortFlags; /* Mask of KEYINFO_ORDER_* flags */ ++ unsigned eEName :2; /* Meaning of zEName */ + unsigned done :1; /* A flag to indicate when processing is finished */ +- unsigned bSpanIsTab :1; /* zSpan holds DB.TABLE.COLUMN */ + unsigned reusable :1; /* Constant expression is reusable */ + unsigned bSorterRef :1; /* Defer evaluation until after sorting */ ++ unsigned bNulls: 1; /* True if explicit "NULLS FIRST/LAST" */ + union { + struct { + u16 iOrderByCol; /* For ORDER BY, column number in result set */ +@@ -17531,6 +18168,13 @@ + } a[1]; /* One slot for each expression in the list */ + }; + ++/* ++** Allowed values for Expr.a.eEName ++*/ ++#define ENAME_NAME 0 /* The AS clause of a result set */ ++#define ENAME_SPAN 1 /* Complete text of the result set expression */ ++#define ENAME_TAB 2 /* "DB.TABLE.NAME" for the result set */ ++ + /* + ** An instance of this structure can hold a simple list of identifiers, + ** such as the list "a,b,c" in the following statements: +@@ -17594,6 +18238,7 @@ + unsigned isCorrelated :1; /* True if sub-query is correlated */ + unsigned viaCoroutine :1; /* Implemented as a co-routine */ + unsigned isRecursive :1; /* True for recursive reference in WITH */ ++ unsigned fromDDL :1; /* Comes from sqlite_master */ + } fg; + int iCursor; /* The VDBE cursor number used to access this table */ + Expr *pOn; /* The ON clause of a join */ +@@ -17684,7 +18329,7 @@ + NameContext *pNext; /* Next outer name context. NULL for outermost */ + int nRef; /* Number of names resolved by this context */ + int nErr; /* Number of errors encountered while resolving names */ +- u16 ncFlags; /* Zero or more NC_* flags defined below */ ++ int ncFlags; /* Zero or more NC_* flags defined below */ + Select *pWinSelect; /* SELECT statement for any window functions */ + }; + +@@ -17697,20 +18342,24 @@ + ** NC_HasWin == EP_Win + ** + */ +-#define NC_AllowAgg 0x0001 /* Aggregate functions are allowed here */ +-#define NC_PartIdx 0x0002 /* True if resolving a partial index WHERE */ +-#define NC_IsCheck 0x0004 /* True if resolving names in a CHECK constraint */ +-#define NC_InAggFunc 0x0008 /* True if analyzing arguments to an agg func */ +-#define NC_HasAgg 0x0010 /* One or more aggregate functions seen */ +-#define NC_IdxExpr 0x0020 /* True if resolving columns of CREATE INDEX */ +-#define NC_VarSelect 0x0040 /* A correlated subquery has been seen */ +-#define NC_UEList 0x0080 /* True if uNC.pEList is used */ +-#define NC_UAggInfo 0x0100 /* True if uNC.pAggInfo is used */ +-#define NC_UUpsert 0x0200 /* True if uNC.pUpsert is used */ +-#define NC_MinMaxAgg 0x1000 /* min/max aggregates seen. See note above */ +-#define NC_Complex 0x2000 /* True if a function or subquery seen */ +-#define NC_AllowWin 0x4000 /* Window functions are allowed here */ +-#define NC_HasWin 0x8000 /* One or more window functions seen */ ++#define NC_AllowAgg 0x00001 /* Aggregate functions are allowed here */ ++#define NC_PartIdx 0x00002 /* True if resolving a partial index WHERE */ ++#define NC_IsCheck 0x00004 /* True if resolving a CHECK constraint */ ++#define NC_GenCol 0x00008 /* True for a GENERATED ALWAYS AS clause */ ++#define NC_HasAgg 0x00010 /* One or more aggregate functions seen */ ++#define NC_IdxExpr 0x00020 /* True if resolving columns of CREATE INDEX */ ++#define NC_SelfRef 0x0002e /* Combo: PartIdx, isCheck, GenCol, and IdxExpr */ ++#define NC_VarSelect 0x00040 /* A correlated subquery has been seen */ ++#define NC_UEList 0x00080 /* True if uNC.pEList is used */ ++#define NC_UAggInfo 0x00100 /* True if uNC.pAggInfo is used */ ++#define NC_UUpsert 0x00200 /* True if uNC.pUpsert is used */ ++#define NC_MinMaxAgg 0x01000 /* min/max aggregates seen. See note above */ ++#define NC_Complex 0x02000 /* True if a function or subquery seen */ ++#define NC_AllowWin 0x04000 /* Window functions are allowed here */ ++#define NC_HasWin 0x08000 /* One or more window functions seen */ ++#define NC_IsDDL 0x10000 /* Resolving names in a CREATE statement */ ++#define NC_InAggFunc 0x20000 /* True if analyzing arguments to an agg func */ ++#define NC_FromDDL 0x40000 /* SQL text comes from sqlite_master */ + + /* + ** An instance of the following object describes a single ON CONFLICT +@@ -17760,13 +18409,13 @@ + ** sequences for the ORDER BY clause. + */ + struct Select { +- ExprList *pEList; /* The fields of the result */ + u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ + LogEst nSelectRow; /* Estimated number of result rows */ + u32 selFlags; /* Various SF_* values */ + int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ + u32 selId; /* Unique identifier number for this SELECT */ + int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ ++ ExprList *pEList; /* The fields of the result */ + SrcList *pSrc; /* The FROM clause */ + Expr *pWhere; /* The WHERE clause */ + ExprList *pGroupBy; /* The GROUP BY clause */ +@@ -17791,25 +18440,28 @@ + ** SF_MinMaxAgg == NC_MinMaxAgg == SQLITE_FUNC_MINMAX + ** SF_FixedLimit == WHERE_USE_LIMIT + */ +-#define SF_Distinct 0x00001 /* Output should be DISTINCT */ +-#define SF_All 0x00002 /* Includes the ALL keyword */ +-#define SF_Resolved 0x00004 /* Identifiers have been resolved */ +-#define SF_Aggregate 0x00008 /* Contains agg functions or a GROUP BY */ +-#define SF_HasAgg 0x00010 /* Contains aggregate functions */ +-#define SF_UsesEphemeral 0x00020 /* Uses the OpenEphemeral opcode */ +-#define SF_Expanded 0x00040 /* sqlite3SelectExpand() called on this */ +-#define SF_HasTypeInfo 0x00080 /* FROM subqueries have Table metadata */ +-#define SF_Compound 0x00100 /* Part of a compound query */ +-#define SF_Values 0x00200 /* Synthesized from VALUES clause */ +-#define SF_MultiValue 0x00400 /* Single VALUES term with multiple rows */ +-#define SF_NestedFrom 0x00800 /* Part of a parenthesized FROM clause */ +-#define SF_MinMaxAgg 0x01000 /* Aggregate containing min() or max() */ +-#define SF_Recursive 0x02000 /* The recursive part of a recursive CTE */ +-#define SF_FixedLimit 0x04000 /* nSelectRow set by a constant LIMIT */ +-#define SF_MaybeConvert 0x08000 /* Need convertCompoundSelectToSubquery() */ +-#define SF_Converted 0x10000 /* By convertCompoundSelectToSubquery() */ +-#define SF_IncludeHidden 0x20000 /* Include hidden columns in output */ +-#define SF_ComplexResult 0x40000 /* Result contains subquery or function */ ++#define SF_Distinct 0x0000001 /* Output should be DISTINCT */ ++#define SF_All 0x0000002 /* Includes the ALL keyword */ ++#define SF_Resolved 0x0000004 /* Identifiers have been resolved */ ++#define SF_Aggregate 0x0000008 /* Contains agg functions or a GROUP BY */ ++#define SF_HasAgg 0x0000010 /* Contains aggregate functions */ ++#define SF_UsesEphemeral 0x0000020 /* Uses the OpenEphemeral opcode */ ++#define SF_Expanded 0x0000040 /* sqlite3SelectExpand() called on this */ ++#define SF_HasTypeInfo 0x0000080 /* FROM subqueries have Table metadata */ ++#define SF_Compound 0x0000100 /* Part of a compound query */ ++#define SF_Values 0x0000200 /* Synthesized from VALUES clause */ ++#define SF_MultiValue 0x0000400 /* Single VALUES term with multiple rows */ ++#define SF_NestedFrom 0x0000800 /* Part of a parenthesized FROM clause */ ++#define SF_MinMaxAgg 0x0001000 /* Aggregate containing min() or max() */ ++#define SF_Recursive 0x0002000 /* The recursive part of a recursive CTE */ ++#define SF_FixedLimit 0x0004000 /* nSelectRow set by a constant LIMIT */ ++#define SF_MaybeConvert 0x0008000 /* Need convertCompoundSelectToSubquery() */ ++#define SF_Converted 0x0010000 /* By convertCompoundSelectToSubquery() */ ++#define SF_IncludeHidden 0x0020000 /* Include hidden columns in output */ ++#define SF_ComplexResult 0x0040000 /* Result contains subquery or function */ ++#define SF_WhereBegin 0x0080000 /* Really a WhereBegin() call. Debug Only */ ++#define SF_WinRewrite 0x0100000 /* Window function rewrite accomplished */ ++#define SF_View 0x0200000 /* SELECT statement is a view */ + + /* + ** The results of a SELECT can be distributed in several ways, as defined +@@ -18089,8 +18741,8 @@ + + #define PARSE_MODE_NORMAL 0 + #define PARSE_MODE_DECLARE_VTAB 1 +-#define PARSE_MODE_RENAME_COLUMN 2 +-#define PARSE_MODE_RENAME_TABLE 3 ++#define PARSE_MODE_RENAME 2 ++#define PARSE_MODE_UNMAP 3 + + /* + ** Sizes and pointers of various parts of the Parse object. +@@ -18112,7 +18764,7 @@ + #if defined(SQLITE_OMIT_ALTERTABLE) + #define IN_RENAME_OBJECT 0 + #else +- #define IN_RENAME_OBJECT (pParse->eParseMode>=PARSE_MODE_RENAME_COLUMN) ++ #define IN_RENAME_OBJECT (pParse->eParseMode>=PARSE_MODE_RENAME) + #endif + + #if defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_OMIT_ALTERTABLE) +@@ -18263,7 +18915,7 @@ + struct DbFixer { + Parse *pParse; /* The parsing context. Error messages written here */ + Schema *pSchema; /* Fix items to this schema */ +- int bVarOnly; /* Check for variable references only */ ++ u8 bTemp; /* True for TEMP schema entries */ + const char *zDb; /* Make sure all objects are contained in this database */ + const char *zType; /* Type of the container - used for error messages */ + const Token *pName; /* Name of the container - used for error messages */ +@@ -18314,11 +18966,12 @@ + */ + struct Sqlite3Config { + int bMemstat; /* True to enable memory status */ +- int bCoreMutex; /* True to enable core mutexing */ +- int bFullMutex; /* True to enable full mutexing */ +- int bOpenUri; /* True to interpret filenames as URIs */ +- int bUseCis; /* Use covering indices for full-scans */ +- int bSmallMalloc; /* Avoid large memory allocations if true */ ++ u8 bCoreMutex; /* True to enable core mutexing */ ++ u8 bFullMutex; /* True to enable full mutexing */ ++ u8 bOpenUri; /* True to interpret filenames as URIs */ ++ u8 bUseCis; /* Use covering indices for full-scans */ ++ u8 bSmallMalloc; /* Avoid large memory allocations if true */ ++ u8 bExtraSchemaChecks; /* Verify type,name,tbl_name in schema */ + int mxStrlen; /* Maximum string length */ + int neverCorrupt; /* Database is always well-formed */ + int szLookaside; /* Default lookaside buffer size */ +@@ -18367,9 +19020,9 @@ + int (*xTestCallback)(int); /* Invoked by sqlite3FaultSim() */ + #endif + int bLocaltimeFault; /* True to fail localtime() calls */ +- int bInternalFunctions; /* Internal SQL functions are visible */ + int iOnceResetThreshold; /* When to reset OP_Once counters */ + u32 szSorterRef; /* Min size in bytes to use sorter-refs */ ++ unsigned int iPrngSeed; /* Alternative fixed seed for the PRNG */ + }; + + /* +@@ -18399,7 +19052,7 @@ + int (*xSelectCallback)(Walker*,Select*); /* Callback for SELECTs */ + void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ + int walkerDepth; /* Number of subqueries */ +- u8 eCode; /* A small processing code */ ++ u16 eCode; /* A small processing code */ + union { /* Extra data for callback */ + NameContext *pNC; /* Naming context */ + int n; /* A counter */ +@@ -18415,6 +19068,8 @@ + struct WindowRewrite *pRewrite; /* Window rewrite context */ + struct WhereConst *pConst; /* WHERE clause constants */ + struct RenameCtx *pRename; /* RENAME COLUMN context */ ++ struct Table *pTab; /* Table of generated column */ ++ struct SrcList_item *pSrcItem; /* A single FROM clause item */ + } u; + }; + +@@ -18427,6 +19082,9 @@ + SQLITE_PRIVATE int sqlite3ExprWalkNoop(Walker*, Expr*); + SQLITE_PRIVATE int sqlite3SelectWalkNoop(Walker*, Select*); + SQLITE_PRIVATE int sqlite3SelectWalkFail(Walker*, Select*); ++SQLITE_PRIVATE int sqlite3WalkerDepthIncrease(Walker*,Select*); ++SQLITE_PRIVATE void sqlite3WalkerDepthDecrease(Walker*,Select*); ++ + #ifdef SQLITE_DEBUG + SQLITE_PRIVATE void sqlite3SelectWalkAssert2(Walker*, Select*); + #endif +@@ -18466,10 +19124,11 @@ + #endif /* SQLITE_DEBUG */ + + /* +-** This object is used in various ways, all related to window functions ++** This object is used in various ways, most (but not all) related to window ++** functions. + ** + ** (1) A single instance of this structure is attached to the +-** the Expr.pWin field for each window function in an expression tree. ++** the Expr.y.pWin field for each window function in an expression tree. + ** This object holds the information contained in the OVER clause, + ** plus additional fields used during code generation. + ** +@@ -18480,6 +19139,10 @@ + ** (3) The terms of the WINDOW clause of a SELECT are instances of this + ** object on a linked list attached to Select.pWinDefn. + ** ++** (4) For an aggregate function with a FILTER clause, an instance ++** of this object is stored in Expr.y.pWin with eFrmType set to ++** TK_FILTER. In this case the only field used is Window.pFilter. ++** + ** The uses (1) and (2) are really the same Window object that just happens + ** to be accessible in two different ways. Use case (3) are separate objects. + */ +@@ -18495,12 +19158,13 @@ + u8 eExclude; /* TK_NO, TK_CURRENT, TK_TIES, TK_GROUP, or 0 */ + Expr *pStart; /* Expression for "<expr> PRECEDING" */ + Expr *pEnd; /* Expression for "<expr> FOLLOWING" */ ++ Window **ppThis; /* Pointer to this object in Select.pWin list */ + Window *pNextWin; /* Next window function belonging to this SELECT */ + Expr *pFilter; /* The FILTER expression */ + FuncDef *pFunc; /* The function */ + int iEphCsr; /* Partition buffer or Peer buffer */ +- int regAccum; +- int regResult; ++ int regAccum; /* Accumulator */ ++ int regResult; /* Interim result */ + int csrApp; /* Function cursor (used by min/max) */ + int regApp; /* Function register (also used by min/max) */ + int regPart; /* Array of registers for PARTITION BY values */ +@@ -18510,15 +19174,19 @@ + int regOne; /* Register containing constant value 1 */ + int regStartRowid; + int regEndRowid; ++ u8 bExprArgs; /* Defer evaluation of window function arguments ++ ** due to the SQLITE_SUBTYPE flag */ + }; + + #ifndef SQLITE_OMIT_WINDOWFUNC + SQLITE_PRIVATE void sqlite3WindowDelete(sqlite3*, Window*); ++SQLITE_PRIVATE void sqlite3WindowUnlinkFromSelect(Window*); + SQLITE_PRIVATE void sqlite3WindowListDelete(sqlite3 *db, Window *p); + SQLITE_PRIVATE Window *sqlite3WindowAlloc(Parse*, int, int, Expr*, int , Expr*, u8); + SQLITE_PRIVATE void sqlite3WindowAttach(Parse*, Expr*, Window*); +-SQLITE_PRIVATE int sqlite3WindowCompare(Parse*, Window*, Window*); +-SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse*, Window*); ++SQLITE_PRIVATE void sqlite3WindowLink(Select *pSel, Window *pWin); ++SQLITE_PRIVATE int sqlite3WindowCompare(Parse*, Window*, Window*, int); ++SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse*, Select*); + SQLITE_PRIVATE void sqlite3WindowCodeStep(Parse*, Select*, WhereInfo*, int, int); + SQLITE_PRIVATE int sqlite3WindowRewrite(Parse*, Select*); + SQLITE_PRIVATE int sqlite3ExpandSubquery(Parse*, struct SrcList_item*); +@@ -18561,13 +19229,16 @@ + #ifdef SQLITE_DEBUG + SQLITE_PRIVATE int sqlite3NomemError(int); + SQLITE_PRIVATE int sqlite3IoerrnomemError(int); +-SQLITE_PRIVATE int sqlite3CorruptPgnoError(int,Pgno); + # define SQLITE_NOMEM_BKPT sqlite3NomemError(__LINE__) + # define SQLITE_IOERR_NOMEM_BKPT sqlite3IoerrnomemError(__LINE__) +-# define SQLITE_CORRUPT_PGNO(P) sqlite3CorruptPgnoError(__LINE__,(P)) + #else + # define SQLITE_NOMEM_BKPT SQLITE_NOMEM + # define SQLITE_IOERR_NOMEM_BKPT SQLITE_IOERR_NOMEM ++#endif ++#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_CORRUPT_PGNO) ++SQLITE_PRIVATE int sqlite3CorruptPgnoError(int,Pgno); ++# define SQLITE_CORRUPT_PGNO(P) sqlite3CorruptPgnoError(__LINE__,(P)) ++#else + # define SQLITE_CORRUPT_PGNO(P) sqlite3CorruptError(__LINE__) + #endif + +@@ -18717,8 +19388,12 @@ + #endif + + #ifndef SQLITE_OMIT_FLOATING_POINT ++# define EXP754 (((u64)0x7ff)<<52) ++# define MAN754 ((((u64)1)<<52)-1) ++# define IsNaN(X) (((X)&EXP754)==EXP754 && ((X)&MAN754)!=0) + SQLITE_PRIVATE int sqlite3IsNaN(double); + #else ++# define IsNaN(X) 0 + # define sqlite3IsNaN(X) 0 + #endif + +@@ -18777,13 +19452,16 @@ + SQLITE_PRIVATE void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*); + SQLITE_PRIVATE Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*); + SQLITE_PRIVATE void sqlite3PExprAddSelect(Parse*, Expr*, Select*); +-SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3*,Expr*, Expr*); ++SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse*,Expr*, Expr*); ++SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr*); + SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*, int); ++SQLITE_PRIVATE void sqlite3ExprFunctionUsable(Parse*,Expr*,FuncDef*); + SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32); + SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*); ++SQLITE_PRIVATE void sqlite3ExprUnmapAndDelete(Parse*, Expr*); + SQLITE_PRIVATE ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); + SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector(Parse*,ExprList*,IdList*,Expr*); +-SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList*,int); ++SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList*,int,int); + SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int); + SQLITE_PRIVATE void sqlite3ExprListSetSpan(Parse*,ExprList*,const char*,const char*); + SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3*, ExprList*); +@@ -18802,11 +19480,18 @@ + SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3*); + SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3*,Table*); + SQLITE_PRIVATE int sqlite3ColumnsFromExprList(Parse*,ExprList*,i16*,Column**); +-SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation(Parse*,Table*,Select*); +-SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse*,Select*); ++SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation(Parse*,Table*,Select*,char); ++SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse*,Select*,char); + SQLITE_PRIVATE void sqlite3OpenMasterTable(Parse *, int); + SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table*); +-SQLITE_PRIVATE i16 sqlite3ColumnOfIndex(Index*, i16); ++SQLITE_PRIVATE i16 sqlite3TableColumnToIndex(Index*, i16); ++#ifdef SQLITE_OMIT_GENERATED_COLUMNS ++# define sqlite3TableColumnToStorage(T,X) (X) /* No-op pass-through */ ++# define sqlite3StorageColumnToTable(T,X) (X) /* No-op pass-through */ ++#else ++SQLITE_PRIVATE i16 sqlite3TableColumnToStorage(Table*, i16); ++SQLITE_PRIVATE i16 sqlite3StorageColumnToTable(Table*, i16); ++#endif + SQLITE_PRIVATE void sqlite3StartTable(Parse*,Token*,Token*,int,int,int,int); + #if SQLITE_ENABLE_HIDDEN_COLUMNS + SQLITE_PRIVATE void sqlite3ColumnPropertiesFromName(Table*, Column*); +@@ -18819,14 +19504,11 @@ + SQLITE_PRIVATE void sqlite3AddCheckConstraint(Parse*, Expr*); + SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse*,Expr*,const char*,const char*); + SQLITE_PRIVATE void sqlite3AddCollateType(Parse*, Token*); ++SQLITE_PRIVATE void sqlite3AddGenerated(Parse*,Expr*,Token*); + SQLITE_PRIVATE void sqlite3EndTable(Parse*,Token*,Token*,u8,Select*); + SQLITE_PRIVATE int sqlite3ParseUri(const char*,const char*,unsigned int*, + sqlite3_vfs**,char**,char **); +-#ifdef SQLITE_HAS_CODEC +-SQLITE_PRIVATE int sqlite3CodecQueryParameters(sqlite3*,const char*,const char*); +-#else +-# define sqlite3CodecQueryParameters(A,B,C) 0 +-#endif ++#define sqlite3CodecQueryParameters(A,B,C) 0 + SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3*,const char*); + + #ifdef SQLITE_UNTESTABLE +@@ -18876,6 +19558,9 @@ + # define sqlite3AutoincrementEnd(X) + #endif + SQLITE_PRIVATE void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int, Upsert*); ++#ifndef SQLITE_OMIT_GENERATED_COLUMNS ++SQLITE_PRIVATE void sqlite3ComputeGeneratedColumns(Parse*, int, Table*); ++#endif + SQLITE_PRIVATE void *sqlite3ArrayAllocate(sqlite3*,void*,int,int*,int*); + SQLITE_PRIVATE IdList *sqlite3IdListAppend(Parse*, IdList*, Token*); + SQLITE_PRIVATE int sqlite3IdListIndex(IdList*,const char*); +@@ -18898,6 +19583,7 @@ + SQLITE_PRIVATE Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*, + Expr*,ExprList*,u32,Expr*); + SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3*, Select*); ++SQLITE_PRIVATE void sqlite3SelectReset(Parse*, Select*); + SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse*, SrcList*); + SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, int); + SQLITE_PRIVATE void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int); +@@ -18920,17 +19606,20 @@ + #define ONEPASS_OFF 0 /* Use of ONEPASS not allowed */ + #define ONEPASS_SINGLE 1 /* ONEPASS valid for a single row update */ + #define ONEPASS_MULTI 2 /* ONEPASS is valid for multiple rows */ ++SQLITE_PRIVATE int sqlite3WhereUsesDeferredSeek(WhereInfo*); + SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn(Parse*, Index*, int, int, int); + SQLITE_PRIVATE int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); + SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); + SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse*, int, int, int); + SQLITE_PRIVATE void sqlite3ExprCode(Parse*, Expr*, int); ++#ifndef SQLITE_OMIT_GENERATED_COLUMNS ++SQLITE_PRIVATE void sqlite3ExprCodeGeneratedColumn(Parse*, Column*, int); ++#endif + SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse*, Expr*, int); + SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse*, Expr*, int); +-SQLITE_PRIVATE int sqlite3ExprCodeAtInit(Parse*, Expr*, int); ++SQLITE_PRIVATE int sqlite3ExprCodeRunJustOnce(Parse*, Expr*, int); + SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse*, Expr*, int*); + SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse*, Expr*, int); +-SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse*, Expr*, int); + SQLITE_PRIVATE int sqlite3ExprCodeExprList(Parse*, ExprList*, int, int, u8); + #define SQLITE_ECEL_DUP 0x01 /* Deep, not shallow copies */ + #define SQLITE_ECEL_FACTOR 0x02 /* Factor out constant terms */ +@@ -18972,6 +19661,7 @@ + SQLITE_PRIVATE void sqlite3Savepoint(Parse*, int, Token*); + SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *); + SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3*); ++SQLITE_PRIVATE u32 sqlite3IsTrueOrFalse(const char*); + SQLITE_PRIVATE int sqlite3ExprIdToTrueFalse(Expr*); + SQLITE_PRIVATE int sqlite3ExprTruthValue(const Expr*); + SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr*); +@@ -19067,6 +19757,7 @@ + #endif + + SQLITE_PRIVATE int sqlite3JoinType(Parse*, Token*, Token*, Token*); ++SQLITE_PRIVATE void sqlite3SetJoinExpr(Expr*,int); + SQLITE_PRIVATE void sqlite3CreateForeignKey(Parse*, ExprList*, Token*, ExprList*, int); + SQLITE_PRIVATE void sqlite3DeferForeignKey(Parse*, int); + #ifndef SQLITE_OMIT_AUTHORIZATION +@@ -19081,6 +19772,7 @@ + # define sqlite3AuthContextPush(a,b,c) + # define sqlite3AuthContextPop(a) ((void)(a)) + #endif ++SQLITE_PRIVATE int sqlite3DbIsNamed(sqlite3 *db, int iDb, const char *zName); + SQLITE_PRIVATE void sqlite3Attach(Parse*, Expr*, Expr*, Expr*); + SQLITE_PRIVATE void sqlite3Detach(Parse*, Expr*); + SQLITE_PRIVATE void sqlite3FixInit(DbFixer*, Parse*, int, const char*, const Token*); +@@ -19089,6 +19781,7 @@ + SQLITE_PRIVATE int sqlite3FixExpr(DbFixer*, Expr*); + SQLITE_PRIVATE int sqlite3FixExprList(DbFixer*, ExprList*); + SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); ++SQLITE_PRIVATE int sqlite3RealSameAsInt(double,sqlite3_int64); + SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8); + SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); + SQLITE_PRIVATE int sqlite3Atoi(const char*); +@@ -19103,7 +19796,7 @@ + SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double); + #endif + #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ +- defined(SQLITE_ENABLE_STAT3_OR_STAT4) || \ ++ defined(SQLITE_ENABLE_STAT4) || \ + defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) + SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst); + #endif +@@ -19128,6 +19821,8 @@ + */ + #define getVarint32(A,B) \ + (u8)((*(A)<(u8)0x80)?((B)=(u32)*(A)),1:sqlite3GetVarint32((A),(u32 *)&(B))) ++#define getVarint32NR(A,B) \ ++ B=(u32)*(A);if(B>=0x80)sqlite3GetVarint32((A),(u32*)&(B)) + #define putVarint32(A,B) \ + (u8)(((u32)(B)<(u32)0x80)?(*(A)=(unsigned char)(B)),1:\ + sqlite3PutVarint((A),(B))) +@@ -19137,10 +19832,10 @@ + + SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3*, Index*); + SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe*, Table*, int); +-SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2); +-SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity); ++SQLITE_PRIVATE char sqlite3CompareAffinity(const Expr *pExpr, char aff2); ++SQLITE_PRIVATE int sqlite3IndexAffinityOk(const Expr *pExpr, char idx_affinity); + SQLITE_PRIVATE char sqlite3TableColumnAffinity(Table*,int); +-SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr); ++SQLITE_PRIVATE char sqlite3ExprAffinity(const Expr *pExpr); + SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*, int, u8); + SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char*, i64*); + SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...); +@@ -19163,15 +19858,17 @@ + SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); + SQLITE_PRIVATE int sqlite3IsBinary(const CollSeq*); + SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName); +-SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr); +-SQLITE_PRIVATE CollSeq *sqlite3ExprNNCollSeq(Parse *pParse, Expr *pExpr); +-SQLITE_PRIVATE int sqlite3ExprCollSeqMatch(Parse*,Expr*,Expr*); ++SQLITE_PRIVATE void sqlite3SetTextEncoding(sqlite3 *db, u8); ++SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr); ++SQLITE_PRIVATE CollSeq *sqlite3ExprNNCollSeq(Parse *pParse, const Expr *pExpr); ++SQLITE_PRIVATE int sqlite3ExprCollSeqMatch(Parse*,const Expr*,const Expr*); + SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, const Token*, int); + SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse*,Expr*,const char*); + SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr*); ++SQLITE_PRIVATE Expr *sqlite3ExprSkipCollateAndLikely(Expr*); + SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *, CollSeq *); + SQLITE_PRIVATE int sqlite3WritableSchema(sqlite3*); +-SQLITE_PRIVATE int sqlite3CheckObjectName(Parse *, const char *); ++SQLITE_PRIVATE int sqlite3CheckObjectName(Parse*, const char*,const char*,const char*); + SQLITE_PRIVATE void sqlite3VdbeSetChanges(sqlite3 *, int); + SQLITE_PRIVATE int sqlite3AddInt64(i64*,i64); + SQLITE_PRIVATE int sqlite3SubInt64(i64*,i64); +@@ -19190,6 +19887,9 @@ + void(*)(void*)); + SQLITE_PRIVATE void sqlite3ValueSetNull(sqlite3_value*); + SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value*); ++#ifndef SQLITE_UNTESTABLE ++SQLITE_PRIVATE void sqlite3ResultIntReal(sqlite3_context*); ++#endif + SQLITE_PRIVATE sqlite3_value *sqlite3ValueNew(sqlite3 *); + #ifndef SQLITE_OMIT_UTF16 + SQLITE_PRIVATE char *sqlite3Utf16to8(sqlite3 *, const void*, int, u8); +@@ -19201,7 +19901,6 @@ + SQLITE_PRIVATE const char sqlite3StrBINARY[]; + SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[]; + SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[]; +-SQLITE_PRIVATE const Token sqlite3IntTokens[]; + SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config; + SQLITE_PRIVATE FuncDefHash sqlite3BuiltinFunctions; + #ifndef SQLITE_OMIT_WSD +@@ -19223,7 +19922,14 @@ + SQLITE_PRIVATE int sqlite3CodeSubselect(Parse*, Expr*); + SQLITE_PRIVATE void sqlite3SelectPrep(Parse*, Select*, NameContext*); + SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p); +-SQLITE_PRIVATE int sqlite3MatchSpanName(const char*, const char*, const char*, const char*); ++SQLITE_PRIVATE int sqlite3MatchEName( ++ const struct ExprList_item*, ++ const char*, ++ const char*, ++ const char* ++); ++SQLITE_PRIVATE Bitmask sqlite3ExprColUsed(Expr*); ++SQLITE_PRIVATE u8 sqlite3StrIHash(const char*); + SQLITE_PRIVATE int sqlite3ResolveExprNames(NameContext*, Expr*); + SQLITE_PRIVATE int sqlite3ResolveExprListNames(NameContext*, ExprList*); + SQLITE_PRIVATE void sqlite3ResolveSelectNames(Parse*, Select*, NameContext*); +@@ -19239,7 +19945,7 @@ + SQLITE_PRIVATE CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*); + SQLITE_PRIVATE char sqlite3AffinityType(const char*, Column*); + SQLITE_PRIVATE void sqlite3Analyze(Parse*, Token*, Token*); +-SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler*, sqlite3_file*); ++SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler*); + SQLITE_PRIVATE int sqlite3FindDb(sqlite3*, Token*); + SQLITE_PRIVATE int sqlite3FindDbName(sqlite3 *, const char *); + SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3*,int iDB); +@@ -19255,6 +19961,7 @@ + SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoRef(KeyInfo*); + SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse*, Index*); + SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoFromExprList(Parse*, ExprList*, int, int); ++SQLITE_PRIVATE int sqlite3HasExplicitNulls(Parse*, ExprList*); + + #ifdef SQLITE_DEBUG + SQLITE_PRIVATE int sqlite3KeyInfoIsWriteable(KeyInfo*); +@@ -19287,8 +19994,7 @@ + # define sqlite3ExprCheckIN(x,y) SQLITE_OK + #endif + +-#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +-SQLITE_PRIVATE void sqlite3AnalyzeFunctions(void); ++#ifdef SQLITE_ENABLE_STAT4 + SQLITE_PRIVATE int sqlite3Stat4ProbeSetValue( + Parse*,Index*,UnpackedRecord**,Expr*,int,int,int*); + SQLITE_PRIVATE int sqlite3Stat4ValueFromExpr(Parse*, Expr*, u8, sqlite3_value**); +@@ -19335,6 +20041,7 @@ + # define sqlite3VtabInSync(db) 0 + # define sqlite3VtabLock(X) + # define sqlite3VtabUnlock(X) ++# define sqlite3VtabModuleUnref(D,X) + # define sqlite3VtabUnlockList(X) + # define sqlite3VtabSavepoint(X, Y, Z) SQLITE_OK + # define sqlite3GetVTable(X,Y) ((VTable*)0) +@@ -19346,6 +20053,7 @@ + SQLITE_PRIVATE int sqlite3VtabCommit(sqlite3 *db); + SQLITE_PRIVATE void sqlite3VtabLock(VTable *); + SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *); ++SQLITE_PRIVATE void sqlite3VtabModuleUnref(sqlite3*,Module*); + SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3*); + SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *, int, int); + SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe*, sqlite3_vtab*); +@@ -19359,6 +20067,14 @@ + ); + # define sqlite3VtabInSync(db) ((db)->nVTrans>0 && (db)->aVTrans==0) + #endif ++SQLITE_PRIVATE int sqlite3ReadOnlyShadowTables(sqlite3 *db); ++#ifndef SQLITE_OMIT_VIRTUALTABLE ++SQLITE_PRIVATE int sqlite3ShadowTableName(sqlite3 *db, const char *zName); ++SQLITE_PRIVATE int sqlite3IsShadowTableOf(sqlite3*,Table*,const char*); ++#else ++# define sqlite3ShadowTableName(A,B) 0 ++# define sqlite3IsShadowTableOf(A,B,C) 0 ++#endif + SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse*,Module*); + SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3*,Module*); + SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse*,Table*); +@@ -19380,7 +20096,8 @@ + #endif + SQLITE_PRIVATE int sqlite3Reprepare(Vdbe*); + SQLITE_PRIVATE void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*); +-SQLITE_PRIVATE CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *); ++SQLITE_PRIVATE CollSeq *sqlite3ExprCompareCollSeq(Parse*,const Expr*); ++SQLITE_PRIVATE CollSeq *sqlite3BinaryCompareCollSeq(Parse *, const Expr*, const Expr*); + SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3*); + SQLITE_PRIVATE const char *sqlite3JournalModename(int); + #ifndef SQLITE_OMIT_WAL +@@ -19686,7 +20403,6 @@ + ** non-ASCII UTF character. Hence the test for whether or not a character is + ** part of an identifier is 0x46. + */ +-#ifdef SQLITE_ASCII + SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 00..07 ........ */ + 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */ +@@ -19724,7 +20440,6 @@ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* f0..f7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40 /* f8..ff ........ */ + }; +-#endif + + /* EVIDENCE-OF: R-02982-34736 In order to maintain full backwards + ** compatibility for legacy applications, the URI filename capability is +@@ -19736,24 +20451,24 @@ + ** EVIDENCE-OF: R-43642-56306 By default, URI handling is globally + ** disabled. The default value may be changed by compiling with the + ** SQLITE_USE_URI symbol defined. +-** +-** URI filenames are enabled by default if SQLITE_HAS_CODEC is +-** enabled. + */ + #ifndef SQLITE_USE_URI +-# ifdef SQLITE_HAS_CODEC +-# define SQLITE_USE_URI 1 +-# else +-# define SQLITE_USE_URI 0 +-# endif ++# define SQLITE_USE_URI 0 + #endif + + /* EVIDENCE-OF: R-38720-18127 The default setting is determined by the + ** SQLITE_ALLOW_COVERING_INDEX_SCAN compile-time option, or is "on" if + ** that compile-time option is omitted. + */ +-#ifndef SQLITE_ALLOW_COVERING_INDEX_SCAN ++#if !defined(SQLITE_ALLOW_COVERING_INDEX_SCAN) + # define SQLITE_ALLOW_COVERING_INDEX_SCAN 1 ++#else ++# if !SQLITE_ALLOW_COVERING_INDEX_SCAN ++# error "Compile-time disabling of covering index scan using the\ ++ -DSQLITE_ALLOW_COVERING_INDEX_SCAN=0 option is deprecated.\ ++ Contact SQLite developers if this is a problem for you, and\ ++ delete this #error macro to continue with your build." ++# endif + #endif + + /* The minimum PMA size is set to this value multiplied by the database +@@ -19782,9 +20497,18 @@ + ** changed as start-time using sqlite3_config(SQLITE_CONFIG_LOOKASIDE) + ** or at run-time for an individual database connection using + ** sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE); ++** ++** With the two-size-lookaside enhancement, less lookaside is required. ++** The default configuration of 1200,40 actually provides 30 1200-byte slots ++** and 93 128-byte slots, which is more lookaside than is available ++** using the older 1200,100 configuration without two-size-lookaside. + */ + #ifndef SQLITE_DEFAULT_LOOKASIDE +-# define SQLITE_DEFAULT_LOOKASIDE 1200,100 ++# ifdef SQLITE_OMIT_TWOSIZE_LOOKASIDE ++# define SQLITE_DEFAULT_LOOKASIDE 1200,100 /* 120KB of memory */ ++# else ++# define SQLITE_DEFAULT_LOOKASIDE 1200,40 /* 48KB of memory */ ++# endif + #endif + + +@@ -19806,6 +20530,7 @@ + SQLITE_USE_URI, /* bOpenUri */ + SQLITE_ALLOW_COVERING_INDEX_SCAN, /* bUseCis */ + 0, /* bSmallMalloc */ ++ 1, /* bExtraSchemaChecks */ + 0x7ffffffe, /* mxStrlen */ + 0, /* neverCorrupt */ + SQLITE_DEFAULT_LOOKASIDE, /* szLookaside, nLookaside */ +@@ -19849,9 +20574,9 @@ + 0, /* xTestCallback */ + #endif + 0, /* bLocaltimeFault */ +- 0, /* bInternalFunctions */ + 0x7ffffffe, /* iOnceResetThreshold */ + SQLITE_DEFAULT_SORTERREF_SIZE, /* szSorterRef */ ++ 0, /* iPrngSeed */ + }; + + /* +@@ -19861,14 +20586,6 @@ + */ + SQLITE_PRIVATE FuncDefHash sqlite3BuiltinFunctions; + +-/* +-** Constant tokens for values 0 and 1. +-*/ +-SQLITE_PRIVATE const Token sqlite3IntTokens[] = { +- { "0", 1 }, +- { "1", 1 } +-}; +- + #ifdef VDBE_PROFILE + /* + ** The following performance counter can be used in place of +@@ -19966,7 +20683,8 @@ + ** "explain" P4 display logic is enabled. + */ + #if !defined(SQLITE_OMIT_EXPLAIN) || !defined(NDEBUG) \ +- || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) ++ || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) \ ++ || defined(SQLITE_ENABLE_BYTECODE_VTAB) + # define VDBE_DISPLAY_P4 1 + #else + # define VDBE_DISPLAY_P4 0 +@@ -20180,12 +20898,12 @@ + #define MEM_Int 0x0004 /* Value is an integer */ + #define MEM_Real 0x0008 /* Value is a real number */ + #define MEM_Blob 0x0010 /* Value is a BLOB */ +-#define MEM_AffMask 0x001f /* Mask of affinity bits */ +-#define MEM_FromBind 0x0020 /* Value originates from sqlite3_bind() */ +-/* Available 0x0040 */ ++#define MEM_IntReal 0x0020 /* MEM_Int that stringifies like MEM_Real */ ++#define MEM_AffMask 0x003f /* Mask of affinity bits */ ++#define MEM_FromBind 0x0040 /* Value originates from sqlite3_bind() */ + #define MEM_Undefined 0x0080 /* Value is undefined */ + #define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */ +-#define MEM_TypeMask 0xc1df /* Mask of type bits */ ++#define MEM_TypeMask 0xc1bf /* Mask of type bits */ + + + /* Whenever Mem contains a valid string or blob representation, one of +@@ -20221,7 +20939,8 @@ + ** True if Mem X is a NULL-nochng type. + */ + #define MemNullNochng(X) \ +- ((X)->flags==(MEM_Null|MEM_Zero) && (X)->n==0 && (X)->u.nZero==0) ++ (((X)->flags&MEM_TypeMask)==(MEM_Null|MEM_Zero) \ ++ && (X)->n==0 && (X)->u.nZero==0) + + /* + ** Return true if a memory cell is not marked as invalid. This macro +@@ -20352,9 +21071,9 @@ + u8 errorAction; /* Recovery action to do in case of an error */ + u8 minWriteFileFormat; /* Minimum file format for writable database files */ + u8 prepFlags; /* SQLITE_PREPARE_* flags */ ++ u8 doingRerun; /* True if rerunning after an auto-reprepare */ + bft expired:2; /* 1: recompile VM immediately 2: when convenient */ + bft explain:2; /* True if EXPLAIN present on SQL command */ +- bft doingRerun:1; /* True if rerunning after an auto-reprepare */ + bft changeCntOn:1; /* True to update the change-counter */ + bft runOnlyOnce:1; /* Automatically expire on reset */ + bft usesStmtJournal:1; /* True if uses a statement journal */ +@@ -20417,11 +21136,11 @@ + SQLITE_PRIVATE void sqlite3VdbeError(Vdbe*, const char *, ...); + SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*); + void sqliteVdbePopStack(Vdbe*,int); ++SQLITE_PRIVATE int SQLITE_NOINLINE sqlite3VdbeFinishMoveto(VdbeCursor*); + SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor**, int*); + SQLITE_PRIVATE int sqlite3VdbeCursorRestore(VdbeCursor*); + SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32); + SQLITE_PRIVATE u8 sqlite3VdbeOneByteSerialTypeLen(u8); +-SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem*, int, u32*); + SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(unsigned char*, Mem*, u32); + SQLITE_PRIVATE u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); + SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3*, AuxData**, int, int); +@@ -20430,7 +21149,14 @@ + SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare(sqlite3*,VdbeCursor*,UnpackedRecord*,int*); + SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3*, BtCursor*, i64*); + SQLITE_PRIVATE int sqlite3VdbeExec(Vdbe*); +-#ifndef SQLITE_OMIT_EXPLAIN ++#if !defined(SQLITE_OMIT_EXPLAIN) || defined(SQLITE_ENABLE_BYTECODE_VTAB) ++SQLITE_PRIVATE int sqlite3VdbeNextOpcode(Vdbe*,Mem*,int,int*,int*,Op**); ++SQLITE_PRIVATE char *sqlite3VdbeDisplayP4(sqlite3*,Op*); ++#endif ++#if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) ++SQLITE_PRIVATE char *sqlite3VdbeDisplayComment(sqlite3*,const Op*,const char*); ++#endif ++#if !defined(SQLITE_OMIT_EXPLAIN) + SQLITE_PRIVATE int sqlite3VdbeList(Vdbe*); + #endif + SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe*); +@@ -20464,14 +21190,15 @@ + SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem*); + SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem*); + SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem*); +-SQLITE_PRIVATE void sqlite3VdbeMemCast(Mem*,u8,u8); ++SQLITE_PRIVATE int sqlite3VdbeMemCast(Mem*,u8,u8); + SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor*,u32,u32,Mem*); ++SQLITE_PRIVATE int sqlite3VdbeMemFromBtreeZeroOffset(BtCursor*,u32,Mem*); + SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p); + SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem*, FuncDef*); + #ifndef SQLITE_OMIT_WINDOWFUNC + SQLITE_PRIVATE int sqlite3VdbeMemAggValue(Mem*, Mem*, FuncDef*); + #endif +-#ifndef SQLITE_OMIT_EXPLAIN ++#if !defined(SQLITE_OMIT_EXPLAIN) || defined(SQLITE_ENABLE_BYTECODE_VTAB) + SQLITE_PRIVATE const char *sqlite3OpcodeName(int); + #endif + SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); +@@ -20530,7 +21257,7 @@ + + #ifdef SQLITE_DEBUG + SQLITE_PRIVATE void sqlite3VdbePrintSql(Vdbe*); +-SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf); ++SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, StrAccum *pStr); + #endif + #ifndef SQLITE_OMIT_UTF16 + SQLITE_PRIVATE int sqlite3VdbeMemTranslate(Mem*, u8); +@@ -20722,6 +21449,10 @@ + SQLITE_PRIVATE int sqlite3LookasideUsed(sqlite3 *db, int *pHighwater){ + u32 nInit = countLookasideSlots(db->lookaside.pInit); + u32 nFree = countLookasideSlots(db->lookaside.pFree); ++#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE ++ nInit += countLookasideSlots(db->lookaside.pSmallInit); ++ nFree += countLookasideSlots(db->lookaside.pSmallFree); ++#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + if( pHighwater ) *pHighwater = db->lookaside.nSlot - nInit; + return db->lookaside.nSlot - (nInit+nFree); + } +@@ -20754,6 +21485,15 @@ + db->lookaside.pInit = db->lookaside.pFree; + db->lookaside.pFree = 0; + } ++#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE ++ p = db->lookaside.pSmallFree; ++ if( p ){ ++ while( p->pNext ) p = p->pNext; ++ p->pNext = db->lookaside.pSmallInit; ++ db->lookaside.pSmallInit = db->lookaside.pSmallFree; ++ db->lookaside.pSmallFree = 0; ++ } ++#endif + } + break; + } +@@ -21305,7 +22045,7 @@ + return 0; + }else if( sqlite3StrICmp(zDate,"now")==0 && sqlite3NotPureFunc(context) ){ + return setDateTimeToCurrent(context, p); +- }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8) ){ ++ }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8)>0 ){ + setRawDateNumber(p, r); + return 0; + } +@@ -21538,12 +22278,12 @@ + double rLimit; /* Maximum NNN value for this transform */ + double rXform; /* Constant used for this transform */ + } aXformType[] = { +- { 0, 6, "second", 464269060800.0, 86400000.0/(24.0*60.0*60.0) }, +- { 0, 6, "minute", 7737817680.0, 86400000.0/(24.0*60.0) }, +- { 0, 4, "hour", 128963628.0, 86400000.0/24.0 }, +- { 0, 3, "day", 5373485.0, 86400000.0 }, +- { 1, 5, "month", 176546.0, 30.0*86400000.0 }, +- { 2, 4, "year", 14713.0, 365.0*86400000.0 }, ++ { 0, 6, "second", 464269060800.0, 1000.0 }, ++ { 0, 6, "minute", 7737817680.0, 60000.0 }, ++ { 0, 4, "hour", 128963628.0, 3600000.0 }, ++ { 0, 3, "day", 5373485.0, 86400000.0 }, ++ { 1, 5, "month", 176546.0, 2592000000.0 }, ++ { 2, 4, "year", 14713.0, 31536000000.0 }, + }; + + /* +@@ -21605,7 +22345,7 @@ + r = p->s*1000.0 + 210866760000000.0; + if( r>=0.0 && r<464269060800000.0 ){ + clearYMD_HMS_TZ(p); +- p->iJD = (sqlite3_int64)r; ++ p->iJD = (sqlite3_int64)(r + 0.5); + p->validJD = 1; + p->rawS = 0; + rc = 0; +@@ -21639,7 +22379,7 @@ + ** date is already on the appropriate weekday, this is a no-op. + */ + if( sqlite3_strnicmp(z, "weekday ", 8)==0 +- && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8) ++ && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8)>0 + && (n=(int)r)==r && n>=0 && r<7 ){ + sqlite3_int64 Z; + computeYMD_HMS(p); +@@ -21698,7 +22438,7 @@ + double rRounder; + int i; + for(n=1; z[n] && z[n]!=':' && !sqlite3Isspace(z[n]); n++){} +- if( !sqlite3AtoF(z, &r, n, SQLITE_UTF8) ){ ++ if( sqlite3AtoF(z, &r, n, SQLITE_UTF8)<=0 ){ + rc = 1; + break; + } +@@ -22384,7 +23124,7 @@ + ** down into the VFS layer. Some SQLITE_OPEN_ flags (for example, + ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before + ** reaching the VFS. */ +- rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x87f7f, pFlagsOut); ++ rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x1087f7f, pFlagsOut); + assert( rc==SQLITE_OK || pFile->pMethods==0 ); + return rc; + } +@@ -22427,7 +23167,15 @@ + } + #endif /* SQLITE_OMIT_LOAD_EXTENSION */ + SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ +- return pVfs->xRandomness(pVfs, nByte, zBufOut); ++ if( sqlite3Config.iPrngSeed ){ ++ memset(zBufOut, 0, nByte); ++ if( ALWAYS(nByte>(signed)sizeof(unsigned)) ) nByte = sizeof(unsigned int); ++ memcpy(zBufOut, &sqlite3Config.iPrngSeed, nByte); ++ return SQLITE_OK; ++ }else{ ++ return pVfs->xRandomness(pVfs, nByte, zBufOut); ++ } ++ + } + SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *pVfs, int nMicro){ + return pVfs->xSleep(pVfs, nMicro); +@@ -25095,6 +25843,7 @@ + GLOBAL(int, mutexIsInit) = 1; + #endif + ++ sqlite3MemoryBarrier(); + return rc; + } + +@@ -25894,7 +26643,7 @@ + ****************************************************************************** + ** + ** This file contains inline asm code for retrieving "high-performance" +-** counters for x86 class CPUs. ++** counters for x86 and x86_64 class CPUs. + */ + #ifndef SQLITE_HWTIME_H + #define SQLITE_HWTIME_H +@@ -25905,8 +26654,9 @@ + ** processor and returns that value. This can be used for high-res + ** profiling. + */ +-#if (defined(__GNUC__) || defined(_MSC_VER)) && \ +- (defined(i386) || defined(__i386__) || defined(_M_IX86)) ++#if !defined(__STRICT_ANSI__) && \ ++ (defined(__GNUC__) || defined(_MSC_VER)) && \ ++ (defined(i386) || defined(__i386__) || defined(_M_IX86)) + + #if defined(__GNUC__) + +@@ -25927,7 +26677,7 @@ + + #endif + +-#elif (defined(__GNUC__) && defined(__x86_64__)) ++#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long val; +@@ -25935,7 +26685,7 @@ + return val; + } + +-#elif (defined(__GNUC__) && defined(__ppc__)) ++#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long long retval; +@@ -25952,14 +26702,13 @@ + + #else + +- #error Need implementation of sqlite3Hwtime() for your platform. +- + /* +- ** To compile without implementing sqlite3Hwtime() for your platform, +- ** you can remove the above #error and use the following +- ** stub function. You will lose timing support for many +- ** of the debugging and testing utilities, but it should at +- ** least compile and run. ++ ** asm() is needed for hardware timing support. Without asm(), ++ ** disable the sqlite3Hwtime() routine. ++ ** ++ ** sqlite3Hwtime() is only used for some obscure debugging ++ ** and analysis configurations, not in any deliverable, so this ++ ** should not be a great loss. + */ + SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } + +@@ -26543,19 +27292,27 @@ + #endif + } + ++/* ++** Default value of the hard heap limit. 0 means "no limit". ++*/ ++#ifndef SQLITE_MAX_MEMORY ++# define SQLITE_MAX_MEMORY 0 ++#endif ++ + /* + ** State information local to the memory allocation subsystem. + */ + static SQLITE_WSD struct Mem0Global { + sqlite3_mutex *mutex; /* Mutex to serialize access */ + sqlite3_int64 alarmThreshold; /* The soft heap limit */ ++ sqlite3_int64 hardLimit; /* The hard upper bound on memory */ + + /* + ** True if heap is nearly "full" where "full" is defined by the + ** sqlite3_soft_heap_limit() setting. + */ + int nearlyFull; +-} mem0 = { 0, 0, 0 }; ++} mem0 = { 0, SQLITE_MAX_MEMORY, SQLITE_MAX_MEMORY, 0 }; + + #define mem0 GLOBAL(struct Mem0Global, mem0) + +@@ -26585,8 +27342,15 @@ + #endif + + /* +-** Set the soft heap-size limit for the library. Passing a zero or +-** negative value indicates no limit. ++** Set the soft heap-size limit for the library. An argument of ++** zero disables the limit. A negative argument is a no-op used to ++** obtain the return value. ++** ++** The return value is the value of the heap limit just before this ++** interface was called. ++** ++** If the hard heap limit is enabled, then the soft heap limit cannot ++** be disabled nor raised above the hard heap limit. + */ + SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 n){ + sqlite3_int64 priorLimit; +@@ -26602,9 +27366,12 @@ + sqlite3_mutex_leave(mem0.mutex); + return priorLimit; + } ++ if( mem0.hardLimit>0 && (n>mem0.hardLimit || n==0) ){ ++ n = mem0.hardLimit; ++ } + mem0.alarmThreshold = n; + nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); +- mem0.nearlyFull = (n>0 && n<=nUsed); ++ AtomicStore(&mem0.nearlyFull, n>0 && n<=nUsed); + sqlite3_mutex_leave(mem0.mutex); + excess = sqlite3_memory_used() - n; + if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff)); +@@ -26615,6 +27382,37 @@ + sqlite3_soft_heap_limit64(n); + } + ++/* ++** Set the hard heap-size limit for the library. An argument of zero ++** disables the hard heap limit. A negative argument is a no-op used ++** to obtain the return value without affecting the hard heap limit. ++** ++** The return value is the value of the hard heap limit just prior to ++** calling this interface. ++** ++** Setting the hard heap limit will also activate the soft heap limit ++** and constrain the soft heap limit to be no more than the hard heap ++** limit. ++*/ ++SQLITE_API sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 n){ ++ sqlite3_int64 priorLimit; ++#ifndef SQLITE_OMIT_AUTOINIT ++ int rc = sqlite3_initialize(); ++ if( rc ) return -1; ++#endif ++ sqlite3_mutex_enter(mem0.mutex); ++ priorLimit = mem0.hardLimit; ++ if( n>=0 ){ ++ mem0.hardLimit = n; ++ if( n<mem0.alarmThreshold || mem0.alarmThreshold==0 ){ ++ mem0.alarmThreshold = n; ++ } ++ } ++ sqlite3_mutex_leave(mem0.mutex); ++ return priorLimit; ++} ++ ++ + /* + ** Initialize the memory allocation subsystem. + */ +@@ -26641,7 +27439,7 @@ + ** sqlite3_soft_heap_limit(). + */ + SQLITE_PRIVATE int sqlite3HeapNearlyFull(void){ +- return mem0.nearlyFull; ++ return AtomicLoad(&mem0.nearlyFull); + } + + /* +@@ -26701,21 +27499,21 @@ + ** following xRoundup() call. */ + nFull = sqlite3GlobalConfig.m.xRoundup(n); + +-#ifdef SQLITE_MAX_MEMORY +- if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)+nFull>SQLITE_MAX_MEMORY ){ +- *pp = 0; +- return; +- } +-#endif +- + sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n); + if( mem0.alarmThreshold>0 ){ + sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + if( nUsed >= mem0.alarmThreshold - nFull ){ +- mem0.nearlyFull = 1; ++ AtomicStore(&mem0.nearlyFull, 1); + sqlite3MallocAlarm(nFull); ++ if( mem0.hardLimit ){ ++ nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); ++ if( nUsed >= mem0.hardLimit - nFull ){ ++ *pp = 0; ++ return; ++ } ++ } + }else{ +- mem0.nearlyFull = 0; ++ AtomicStore(&mem0.nearlyFull, 0); + } + } + p = sqlite3GlobalConfig.m.xMalloc(nFull); +@@ -26794,10 +27592,17 @@ + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + return sqlite3GlobalConfig.m.xSize(p); + } ++static int lookasideMallocSize(sqlite3 *db, void *p){ ++#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE ++ return p<db->lookaside.pMiddle ? db->lookaside.szTrue : LOOKASIDE_SMALL; ++#else ++ return db->lookaside.szTrue; ++#endif ++} + SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, void *p){ + assert( p!=0 ); +- if( db==0 || !isLookaside(db,p) ){ + #ifdef SQLITE_DEBUG ++ if( db==0 || !isLookaside(db,p) ){ + if( db==0 ){ + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); +@@ -26805,12 +27610,23 @@ + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + } ++ } + #endif +- return sqlite3GlobalConfig.m.xSize(p); +- }else{ +- assert( sqlite3_mutex_held(db->mutex) ); +- return db->lookaside.sz; ++ if( db ){ ++ if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ ++#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE ++ if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ ++ assert( sqlite3_mutex_held(db->mutex) ); ++ return LOOKASIDE_SMALL; ++ } ++#endif ++ if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ ++ assert( sqlite3_mutex_held(db->mutex) ); ++ return db->lookaside.szTrue; ++ } ++ } + } ++ return sqlite3GlobalConfig.m.xSize(p); + } + SQLITE_API sqlite3_uint64 sqlite3_msize(void *p){ + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); +@@ -26857,15 +27673,27 @@ + measureAllocationSize(db, p); + return; + } +- if( isLookaside(db, p) ){ +- LookasideSlot *pBuf = (LookasideSlot*)p; ++ if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ ++#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE ++ if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ ++ LookasideSlot *pBuf = (LookasideSlot*)p; + #ifdef SQLITE_DEBUG +- /* Trash all content in the buffer being freed */ +- memset(p, 0xaa, db->lookaside.sz); ++ memset(p, 0xaa, LOOKASIDE_SMALL); /* Trash freed content */ + #endif +- pBuf->pNext = db->lookaside.pFree; +- db->lookaside.pFree = pBuf; +- return; ++ pBuf->pNext = db->lookaside.pSmallFree; ++ db->lookaside.pSmallFree = pBuf; ++ return; ++ } ++#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ ++ if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ ++ LookasideSlot *pBuf = (LookasideSlot*)p; ++#ifdef SQLITE_DEBUG ++ memset(p, 0xaa, db->lookaside.szTrue); /* Trash freed content */ ++#endif ++ pBuf->pNext = db->lookaside.pFree; ++ db->lookaside.pFree = pBuf; ++ return; ++ } + } + } + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); +@@ -26914,10 +27742,12 @@ + sqlite3MallocAlarm(nDiff); + } + pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); ++#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + if( pNew==0 && mem0.alarmThreshold>0 ){ + sqlite3MallocAlarm((int)nBytes); + pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); + } ++#endif + if( pNew ){ + nNew = sqlite3MallocSize(pNew); + sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nNew-nOld); +@@ -27021,23 +27851,37 @@ + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( db->pnBytesFreed==0 ); +- if( db->lookaside.bDisable==0 ){ +- assert( db->mallocFailed==0 ); +- if( n>db->lookaside.sz ){ +- db->lookaside.anStat[1]++; +- }else if( (pBuf = db->lookaside.pFree)!=0 ){ +- db->lookaside.pFree = pBuf->pNext; ++ if( n>db->lookaside.sz ){ ++ if( !db->lookaside.bDisable ){ ++ db->lookaside.anStat[1]++; ++ }else if( db->mallocFailed ){ ++ return 0; ++ } ++ return dbMallocRawFinish(db, n); ++ } ++#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE ++ if( n<=LOOKASIDE_SMALL ){ ++ if( (pBuf = db->lookaside.pSmallFree)!=0 ){ ++ db->lookaside.pSmallFree = pBuf->pNext; + db->lookaside.anStat[0]++; + return (void*)pBuf; +- }else if( (pBuf = db->lookaside.pInit)!=0 ){ +- db->lookaside.pInit = pBuf->pNext; ++ }else if( (pBuf = db->lookaside.pSmallInit)!=0 ){ ++ db->lookaside.pSmallInit = pBuf->pNext; + db->lookaside.anStat[0]++; + return (void*)pBuf; +- }else{ +- db->lookaside.anStat[2]++; + } +- }else if( db->mallocFailed ){ +- return 0; ++ } ++#endif ++ if( (pBuf = db->lookaside.pFree)!=0 ){ ++ db->lookaside.pFree = pBuf->pNext; ++ db->lookaside.anStat[0]++; ++ return (void*)pBuf; ++ }else if( (pBuf = db->lookaside.pInit)!=0 ){ ++ db->lookaside.pInit = pBuf->pNext; ++ db->lookaside.anStat[0]++; ++ return (void*)pBuf; ++ }else{ ++ db->lookaside.anStat[2]++; + } + #else + assert( db!=0 ); +@@ -27061,7 +27905,16 @@ + assert( db!=0 ); + if( p==0 ) return sqlite3DbMallocRawNN(db, n); + assert( sqlite3_mutex_held(db->mutex) ); +- if( isLookaside(db,p) && n<=db->lookaside.sz ) return p; ++ if( ((uptr)p)<(uptr)db->lookaside.pEnd ){ ++#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE ++ if( ((uptr)p)>=(uptr)db->lookaside.pMiddle ){ ++ if( n<=LOOKASIDE_SMALL ) return p; ++ }else ++#endif ++ if( ((uptr)p)>=(uptr)db->lookaside.pStart ){ ++ if( n<=db->lookaside.szTrue ) return p; ++ } ++ } + return dbReallocFinish(db, p, n); + } + static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n){ +@@ -27072,14 +27925,14 @@ + if( isLookaside(db, p) ){ + pNew = sqlite3DbMallocRawNN(db, n); + if( pNew ){ +- memcpy(pNew, p, db->lookaside.sz); ++ memcpy(pNew, p, lookasideMallocSize(db, p)); + sqlite3DbFree(db, p); + } + }else{ + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); +- pNew = sqlite3_realloc64(p, n); ++ pNew = sqlite3Realloc(p, n); + if( !pNew ){ + sqlite3OomFault(db); + } +@@ -27169,9 +28022,9 @@ + if( db->mallocFailed==0 && db->bBenignMalloc==0 ){ + db->mallocFailed = 1; + if( db->nVdbeExec>0 ){ +- db->u1.isInterrupted = 1; ++ AtomicStore(&db->u1.isInterrupted, 1); + } +- db->lookaside.bDisable++; ++ DisableLookaside; + if( db->pParse ){ + db->pParse->rc = SQLITE_NOMEM_BKPT; + } +@@ -27188,9 +28041,9 @@ + SQLITE_PRIVATE void sqlite3OomClear(sqlite3 *db){ + if( db->mallocFailed && db->nVdbeExec==0 ){ + db->mallocFailed = 0; +- db->u1.isInterrupted = 0; ++ AtomicStore(&db->u1.isInterrupted, 0); + assert( db->lookaside.bDisable>0 ); +- db->lookaside.bDisable--; ++ EnableLookaside; + } + } + +@@ -27331,6 +28184,12 @@ + { 'r', 10, 1, etORDINAL, 0, 0 }, + }; + ++/* Floating point constants used for rounding */ ++static const double arRound[] = { ++ 5.0e-01, 5.0e-02, 5.0e-03, 5.0e-04, 5.0e-05, ++ 5.0e-06, 5.0e-07, 5.0e-08, 5.0e-09, 5.0e-10, ++}; ++ + /* + ** If SQLITE_OMIT_FLOATING_POINT is defined, then none of the floating point + ** conversions will work. +@@ -27420,6 +28279,13 @@ + #endif + #define etBUFSIZE SQLITE_PRINT_BUF_SIZE /* Size of the output buffer */ + ++/* ++** Hard limit on the precision of floating-point conversions. ++*/ ++#ifndef SQLITE_PRINTF_PRECISION_LIMIT ++# define SQLITE_FP_PRECISION_LIMIT 100000000 ++#endif ++ + /* + ** Render a string given by "fmt" into the StrAccum object. + */ +@@ -27620,6 +28486,8 @@ + ** xtype The class of the conversion. + ** infop Pointer to the appropriate info struct. + */ ++ assert( width>=0 ); ++ assert( precision>=(-1) ); + switch( xtype ){ + case etPOINTER: + flag_long = sizeof(char*)==sizeof(i64) ? 2 : +@@ -27741,6 +28609,11 @@ + length = 0; + #else + if( precision<0 ) precision = 6; /* Set default precision */ ++#ifdef SQLITE_FP_PRECISION_LIMIT ++ if( precision>SQLITE_FP_PRECISION_LIMIT ){ ++ precision = SQLITE_FP_PRECISION_LIMIT; ++ } ++#endif + if( realvalue<0.0 ){ + realvalue = -realvalue; + prefix = '-'; +@@ -27749,8 +28622,18 @@ + } + if( xtype==etGENERIC && precision>0 ) precision--; + testcase( precision>0xfff ); +- for(idx=precision&0xfff, rounder=0.5; idx>0; idx--, rounder*=0.1){} +- if( xtype==etFLOAT ) realvalue += rounder; ++ idx = precision & 0xfff; ++ rounder = arRound[idx%10]; ++ while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } ++ if( xtype==etFLOAT ){ ++ double rx = (double)realvalue; ++ sqlite3_uint64 u; ++ int ex; ++ memcpy(&u, &rx, sizeof(u)); ++ ex = -1023 + (int)((u>>52)&0x7ff); ++ if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16; ++ realvalue += rounder; ++ } + /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ + exp = 0; + if( sqlite3IsNaN((double)realvalue) ){ +@@ -28013,7 +28896,7 @@ + } + isnull = escarg==0; + if( isnull ) escarg = (xtype==etSQLESCAPE2 ? "NULL" : "(NULL)"); +- /* For %q, %Q, and %w, the precision is the number of byte (or ++ /* For %q, %Q, and %w, the precision is the number of bytes (or + ** characters if the ! flags is present) to use from the input. + ** Because of the extra quoting characters inserted, the number + ** of output characters may be larger than the precision. +@@ -28140,7 +29023,7 @@ + if( p->db ){ + zNew = sqlite3DbRealloc(p->db, zOld, p->nAlloc); + }else{ +- zNew = sqlite3_realloc64(zOld, p->nAlloc); ++ zNew = sqlite3Realloc(zOld, p->nAlloc); + } + if( zNew ){ + assert( p->zText!=0 || p->nChar==0 ); +@@ -28482,7 +29365,7 @@ + SQLITE_PRIVATE void sqlite3DebugPrintf(const char *zFormat, ...){ + va_list ap; + StrAccum acc; +- char zBuf[500]; ++ char zBuf[SQLITE_PRINT_BUF_SIZE*10]; + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); + va_start(ap,zFormat); + sqlite3_str_vappendf(&acc, zFormat, ap); +@@ -28582,7 +29465,7 @@ + va_start(ap, zFormat); + sqlite3_str_vappendf(&acc, zFormat, ap); + va_end(ap); +- assert( acc.nChar>0 ); ++ assert( acc.nChar>0 || acc.accError ); + sqlite3_str_append(&acc, "\n", 1); + } + sqlite3StrAccumFinish(&acc); +@@ -28622,7 +29505,7 @@ + char cSep = '('; + int j; + for(j=0; j<pCte->pCols->nExpr; j++){ +- sqlite3_str_appendf(&x, "%c%s", cSep, pCte->pCols->a[j].zName); ++ sqlite3_str_appendf(&x, "%c%s", cSep, pCte->pCols->a[j].zEName); + cSep = ','; + } + sqlite3_str_appendf(&x, ")"); +@@ -28647,15 +29530,15 @@ + StrAccum x; + char zLine[100]; + sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); +- sqlite3_str_appendf(&x, "{%d,*}", pItem->iCursor); ++ sqlite3_str_appendf(&x, "{%d:*}", pItem->iCursor); + if( pItem->zDatabase ){ + sqlite3_str_appendf(&x, " %s.%s", pItem->zDatabase, pItem->zName); + }else if( pItem->zName ){ + sqlite3_str_appendf(&x, " %s", pItem->zName); + } + if( pItem->pTab ){ +- sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p", +- pItem->pTab->zName, pItem->pTab->nCol, pItem->pTab); ++ sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p used=%llx", ++ pItem->pTab->zName, pItem->pTab->nCol, pItem->pTab, pItem->colUsed); + } + if( pItem->zAlias ){ + sqlite3_str_appendf(&x, " (AS %s)", pItem->zAlias); +@@ -28663,6 +29546,9 @@ + if( pItem->fg.jointype & JT_LEFT ){ + sqlite3_str_appendf(&x, " LEFT-JOIN"); + } ++ if( pItem->fg.fromDDL ){ ++ sqlite3_str_appendf(&x, " DDL"); ++ } + sqlite3StrAccumFinish(&x); + sqlite3TreeViewItem(pView, zLine, i<pSrc->nSrc-1); + if( pItem->pSelect ){ +@@ -28692,13 +29578,17 @@ + sqlite3TreeViewPush(pView, 1); + } + do{ +- sqlite3TreeViewLine(pView, +- "SELECT%s%s (%u/%p) selFlags=0x%x nSelectRow=%d", +- ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), +- ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), +- p->selId, p, p->selFlags, +- (int)p->nSelectRow +- ); ++ if( p->selFlags & SF_WhereBegin ){ ++ sqlite3TreeViewLine(pView, "sqlite3WhereBegin()"); ++ }else{ ++ sqlite3TreeViewLine(pView, ++ "SELECT%s%s (%u/%p) selFlags=0x%x nSelectRow=%d", ++ ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), ++ ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), ++ p->selId, p, p->selFlags, ++ (int)p->nSelectRow ++ ); ++ } + if( cnt++ ) sqlite3TreeViewPop(pView); + if( p->pPrior ){ + n = 1000; +@@ -28715,7 +29605,10 @@ + if( p->pWinDefn ) n++; + #endif + } +- sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); ++ if( p->pEList ){ ++ sqlite3TreeViewExprList(pView, p->pEList, n>0, "result-set"); ++ } ++ n--; + #ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin ){ + Window *pX; +@@ -28904,20 +29797,28 @@ + SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ + const char *zBinOp = 0; /* Binary operator */ + const char *zUniOp = 0; /* Unary operator */ +- char zFlgs[60]; ++ char zFlgs[200]; + pView = sqlite3TreeViewPush(pView, moreToFollow); + if( pExpr==0 ){ + sqlite3TreeViewLine(pView, "nil"); + sqlite3TreeViewPop(pView); + return; + } +- if( pExpr->flags ){ ++ if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags ){ ++ StrAccum x; ++ sqlite3StrAccumInit(&x, 0, zFlgs, sizeof(zFlgs), 0); ++ sqlite3_str_appendf(&x, " fg.af=%x.%c", ++ pExpr->flags, pExpr->affExpr ? pExpr->affExpr : 'n'); + if( ExprHasProperty(pExpr, EP_FromJoin) ){ +- sqlite3_snprintf(sizeof(zFlgs),zFlgs," flags=0x%x iRJT=%d", +- pExpr->flags, pExpr->iRightJoinTable); +- }else{ +- sqlite3_snprintf(sizeof(zFlgs),zFlgs," flags=0x%x",pExpr->flags); ++ sqlite3_str_appendf(&x, " iRJT=%d", pExpr->iRightJoinTable); + } ++ if( ExprHasProperty(pExpr, EP_FromDDL) ){ ++ sqlite3_str_appendf(&x, " DDL"); ++ } ++ if( ExprHasVVAProperty(pExpr, EP_Immutable) ){ ++ sqlite3_str_appendf(&x, " IMMUTABLE"); ++ } ++ sqlite3StrAccumFinish(&x); + }else{ + zFlgs[0] = 0; + } +@@ -28930,10 +29831,18 @@ + case TK_COLUMN: { + if( pExpr->iTable<0 ){ + /* This only happens when coding check constraints */ +- sqlite3TreeViewLine(pView, "COLUMN(%d)%s", pExpr->iColumn, zFlgs); ++ char zOp2[16]; ++ if( pExpr->op2 ){ ++ sqlite3_snprintf(sizeof(zOp2),zOp2," op2=0x%02x",pExpr->op2); ++ }else{ ++ zOp2[0] = 0; ++ } ++ sqlite3TreeViewLine(pView, "COLUMN(%d)%s%s", ++ pExpr->iColumn, zFlgs, zOp2); + }else{ +- sqlite3TreeViewLine(pView, "{%d:%d}%s", +- pExpr->iTable, pExpr->iColumn, zFlgs); ++ sqlite3TreeViewLine(pView, "{%d:%d} pTab=%p%s", ++ pExpr->iTable, pExpr->iColumn, ++ pExpr->y.pTab, zFlgs); + } + if( ExprHasProperty(pExpr, EP_FixedCol) ){ + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); +@@ -29015,6 +29924,7 @@ + case TK_RSHIFT: zBinOp = "RSHIFT"; break; + case TK_CONCAT: zBinOp = "CONCAT"; break; + case TK_DOT: zBinOp = "DOT"; break; ++ case TK_LIMIT: zBinOp = "LIMIT"; break; + + case TK_UMINUS: zUniOp = "UMINUS"; break; + case TK_UPLUS: zUniOp = "UPLUS"; break; +@@ -29030,7 +29940,7 @@ + }; + assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); + assert( pExpr->pRight ); +- assert( pExpr->pRight->op==TK_TRUEFALSE ); ++ assert( sqlite3ExprSkipCollate(pExpr->pRight)->op==TK_TRUEFALSE ); + x = (pExpr->op2==TK_ISNOT)*2 + sqlite3ExprTruthValue(pExpr->pRight); + zUniOp = azOp[x]; + break; +@@ -29043,7 +29953,14 @@ + } + + case TK_COLLATE: { +- sqlite3TreeViewLine(pView, "COLLATE %Q", pExpr->u.zToken); ++ /* COLLATE operators without the EP_Collate flag are intended to ++ ** emulate collation associated with a table column. These show ++ ** up in the treeview output as "SOFT-COLLATE". Explicit COLLATE ++ ** operators that appear in the original SQL always have the ++ ** EP_Collate bit set and appear in treeview output as just "COLLATE" */ ++ sqlite3TreeViewLine(pView, "%sCOLLATE %Q%s", ++ !ExprHasProperty(pExpr, EP_Collate) ? "SOFT-" : "", ++ pExpr->u.zToken, zFlgs); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } +@@ -29058,16 +29975,28 @@ + }else{ + pFarg = pExpr->x.pList; + #ifndef SQLITE_OMIT_WINDOWFUNC +- pWin = pExpr->y.pWin; ++ pWin = ExprHasProperty(pExpr, EP_WinFunc) ? pExpr->y.pWin : 0; + #else + pWin = 0; + #endif + } + if( pExpr->op==TK_AGG_FUNCTION ){ +- sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q", +- pExpr->op2, pExpr->u.zToken); ++ sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q%s iAgg=%d agg=%p", ++ pExpr->op2, pExpr->u.zToken, zFlgs, ++ pExpr->iAgg, pExpr->pAggInfo); ++ }else if( pExpr->op2!=0 ){ ++ const char *zOp2; ++ char zBuf[8]; ++ sqlite3_snprintf(sizeof(zBuf),zBuf,"0x%02x",pExpr->op2); ++ zOp2 = zBuf; ++ if( pExpr->op2==NC_IsCheck ) zOp2 = "NC_IsCheck"; ++ if( pExpr->op2==NC_IdxExpr ) zOp2 = "NC_IdxExpr"; ++ if( pExpr->op2==NC_PartIdx ) zOp2 = "NC_PartIdx"; ++ if( pExpr->op2==NC_GenCol ) zOp2 = "NC_GenCol"; ++ sqlite3TreeViewLine(pView, "FUNCTION %Q%s op2=%s", ++ pExpr->u.zToken, zFlgs, zOp2); + }else{ +- sqlite3TreeViewLine(pView, "FUNCTION %Q", pExpr->u.zToken); ++ sqlite3TreeViewLine(pView, "FUNCTION %Q%s", pExpr->u.zToken, zFlgs); + } + if( pFarg ){ + sqlite3TreeViewExprList(pView, pFarg, pWin!=0, 0); +@@ -29144,7 +30073,7 @@ + #ifndef SQLITE_OMIT_TRIGGER + case TK_RAISE: { + const char *zType = "unk"; +- switch( pExpr->affinity ){ ++ switch( pExpr->affExpr ){ + case OE_Rollback: zType = "rollback"; break; + case OE_Abort: zType = "abort"; break; + case OE_Fail: zType = "fail"; break; +@@ -29161,7 +30090,9 @@ + break; + } + case TK_VECTOR: { +- sqlite3TreeViewBareExprList(pView, pExpr->x.pList, "VECTOR"); ++ char *z = sqlite3_mprintf("VECTOR%s",zFlgs); ++ sqlite3TreeViewBareExprList(pView, pExpr->x.pList, z); ++ sqlite3_free(z); + break; + } + case TK_SELECT_COLUMN: { +@@ -29185,7 +30116,7 @@ + sqlite3TreeViewExpr(pView, pExpr->pRight, 0); + }else if( zUniOp ){ + sqlite3TreeViewLine(pView, "%s%s", zUniOp, zFlgs); +- sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); ++ sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + } + sqlite3TreeViewPop(pView); + } +@@ -29207,8 +30138,9 @@ + sqlite3TreeViewLine(pView, "%s", zLabel); + for(i=0; i<pList->nExpr; i++){ + int j = pList->a[i].u.x.iOrderByCol; +- char *zName = pList->a[i].zName; ++ char *zName = pList->a[i].zEName; + int moreToFollow = i<pList->nExpr - 1; ++ if( pList->a[i].eEName!=ENAME_NAME ) zName = 0; + if( j || zName ){ + sqlite3TreeViewPush(pView, moreToFollow); + moreToFollow = 0; +@@ -29765,26 +30697,6 @@ + } \ + } + +-#define READ_UTF16LE(zIn, TERM, c){ \ +- c = (*zIn++); \ +- c += ((*zIn++)<<8); \ +- if( c>=0xD800 && c<0xE000 && TERM ){ \ +- int c2 = (*zIn++); \ +- c2 += ((*zIn++)<<8); \ +- c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10); \ +- } \ +-} +- +-#define READ_UTF16BE(zIn, TERM, c){ \ +- c = ((*zIn++)<<8); \ +- c += (*zIn++); \ +- if( c>=0xD800 && c<0xE000 && TERM ){ \ +- int c2 = ((*zIn++)<<8); \ +- c2 += (*zIn++); \ +- c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10); \ +- } \ +-} +- + /* + ** Translate a single UTF-8 character. Return the unicode value. + ** +@@ -29875,9 +30787,11 @@ + + #if defined(TRANSLATE_TRACE) && defined(SQLITE_DEBUG) + { +- char zBuf[100]; +- sqlite3VdbeMemPrettyPrint(pMem, zBuf); +- fprintf(stderr, "INPUT: %s\n", zBuf); ++ StrAccum acc; ++ char zBuf[1000]; ++ sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); ++ sqlite3VdbeMemPrettyPrint(pMem, &acc); ++ fprintf(stderr, "INPUT: %s\n", sqlite3StrAccumFinish(&acc)); + } + #endif + +@@ -29959,13 +30873,59 @@ + if( pMem->enc==SQLITE_UTF16LE ){ + /* UTF-16 Little-endian -> UTF-8 */ + while( zIn<zTerm ){ +- READ_UTF16LE(zIn, zIn<zTerm, c); ++ c = *(zIn++); ++ c += (*(zIn++))<<8; ++ if( c>=0xd800 && c<0xe000 ){ ++#ifdef SQLITE_REPLACE_INVALID_UTF ++ if( c>=0xdc00 || zIn>=zTerm ){ ++ c = 0xfffd; ++ }else{ ++ int c2 = *(zIn++); ++ c2 += (*(zIn++))<<8; ++ if( c2<0xdc00 || c2>=0xe000 ){ ++ zIn -= 2; ++ c = 0xfffd; ++ }else{ ++ c = ((c&0x3ff)<<10) + (c2&0x3ff) + 0x10000; ++ } ++ } ++#else ++ if( zIn<zTerm ){ ++ int c2 = (*zIn++); ++ c2 += ((*zIn++)<<8); ++ c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10); ++ } ++#endif ++ } + WRITE_UTF8(z, c); + } + }else{ + /* UTF-16 Big-endian -> UTF-8 */ + while( zIn<zTerm ){ +- READ_UTF16BE(zIn, zIn<zTerm, c); ++ c = (*(zIn++))<<8; ++ c += *(zIn++); ++ if( c>=0xd800 && c<0xe000 ){ ++#ifdef SQLITE_REPLACE_INVALID_UTF ++ if( c>=0xdc00 || zIn>=zTerm ){ ++ c = 0xfffd; ++ }else{ ++ int c2 = (*(zIn++))<<8; ++ c2 += *(zIn++); ++ if( c2<0xdc00 || c2>=0xe000 ){ ++ zIn -= 2; ++ c = 0xfffd; ++ }else{ ++ c = ((c&0x3ff)<<10) + (c2&0x3ff) + 0x10000; ++ } ++ } ++#else ++ if( zIn<zTerm ){ ++ int c2 = ((*zIn++)<<8); ++ c2 += (*zIn++); ++ c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10); ++ } ++#endif ++ } + WRITE_UTF8(z, c); + } + } +@@ -29985,9 +30945,11 @@ + translate_out: + #if defined(TRANSLATE_TRACE) && defined(SQLITE_DEBUG) + { +- char zBuf[100]; +- sqlite3VdbeMemPrettyPrint(pMem, zBuf); +- fprintf(stderr, "OUTPUT: %s\n", zBuf); ++ StrAccum acc; ++ char zBuf[1000]; ++ sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); ++ sqlite3VdbeMemPrettyPrint(pMem, &acc); ++ fprintf(stderr, "OUTPUT: %s\n", sqlite3StrAccumFinish(&acc)); + } + #endif + return SQLITE_OK; +@@ -30122,18 +31084,15 @@ + unsigned char const *z = zIn; + int n = 0; + +- if( SQLITE_UTF16NATIVE==SQLITE_UTF16BE ){ +- while( n<nChar ){ +- READ_UTF16BE(z, 1, c); +- n++; +- } +- }else{ +- while( n<nChar ){ +- READ_UTF16LE(z, 1, c); +- n++; +- } ++ if( SQLITE_UTF16NATIVE==SQLITE_UTF16LE ) z++; ++ while( n<nChar ){ ++ c = z[0]; ++ z += 2; ++ if( c>=0xd8 && c<0xdc && z[0]>=0xdc && z[0]<0xe0 ) z += 2; ++ n++; + } +- return (int)(z-(unsigned char const *)zIn); ++ return (int)(z-(unsigned char const *)zIn) ++ - (SQLITE_UTF16NATIVE==SQLITE_UTF16LE); + } + + #if defined(SQLITE_TEST) +@@ -30163,30 +31122,6 @@ + assert( c==t ); + assert( (z-zBuf)==n ); + } +- for(i=0; i<0x00110000; i++){ +- if( i>=0xD800 && i<0xE000 ) continue; +- z = zBuf; +- WRITE_UTF16LE(z, i); +- n = (int)(z-zBuf); +- assert( n>0 && n<=4 ); +- z[0] = 0; +- z = zBuf; +- READ_UTF16LE(z, 1, c); +- assert( c==i ); +- assert( (z-zBuf)==n ); +- } +- for(i=0; i<0x00110000; i++){ +- if( i>=0xD800 && i<0xE000 ) continue; +- z = zBuf; +- WRITE_UTF16BE(z, i); +- n = (int)(z-zBuf); +- assert( n>0 && n<=4 ); +- z[0] = 0; +- z = zBuf; +- READ_UTF16BE(z, 1, c); +- assert( c==i ); +- assert( (z-zBuf)==n ); +- } + } + #endif /* SQLITE_TEST */ + #endif /* SQLITE_OMIT_UTF16 */ +@@ -30212,8 +31147,8 @@ + */ + /* #include "sqliteInt.h" */ + /* #include <stdarg.h> */ +-#if HAVE_ISNAN || SQLITE_HAVE_ISNAN +-# include <math.h> ++#ifndef SQLITE_OMIT_FLOATING_POINT ++#include <math.h> + #endif + + /* +@@ -30255,47 +31190,11 @@ + #ifndef SQLITE_OMIT_FLOATING_POINT + /* + ** Return true if the floating point value is Not a Number (NaN). +-** +-** Use the math library isnan() function if compiled with SQLITE_HAVE_ISNAN. +-** Otherwise, we have our own implementation that works on most systems. + */ + SQLITE_PRIVATE int sqlite3IsNaN(double x){ +- int rc; /* The value return */ +-#if !SQLITE_HAVE_ISNAN && !HAVE_ISNAN +- /* +- ** Systems that support the isnan() library function should probably +- ** make use of it by compiling with -DSQLITE_HAVE_ISNAN. But we have +- ** found that many systems do not have a working isnan() function so +- ** this implementation is provided as an alternative. +- ** +- ** This NaN test sometimes fails if compiled on GCC with -ffast-math. +- ** On the other hand, the use of -ffast-math comes with the following +- ** warning: +- ** +- ** This option [-ffast-math] should never be turned on by any +- ** -O option since it can result in incorrect output for programs +- ** which depend on an exact implementation of IEEE or ISO +- ** rules/specifications for math functions. +- ** +- ** Under MSVC, this NaN test may fail if compiled with a floating- +- ** point precision mode other than /fp:precise. From the MSDN +- ** documentation: +- ** +- ** The compiler [with /fp:precise] will properly handle comparisons +- ** involving NaN. For example, x != x evaluates to true if x is NaN +- ** ... +- */ +-#ifdef __FAST_MATH__ +-# error SQLite will not work correctly with the -ffast-math option of GCC. +-#endif +- volatile double y = x; +- volatile double z = y; +- rc = (y!=z); +-#else /* if HAVE_ISNAN */ +- rc = isnan(x); +-#endif /* HAVE_ISNAN */ +- testcase( rc ); +- return rc; ++ u64 y; ++ memcpy(&y,&x,sizeof(y)); ++ return IsNaN(y); + } + #endif /* SQLITE_OMIT_FLOATING_POINT */ + +@@ -30425,6 +31324,7 @@ + sqlite3DbFree(db, pParse->zErrMsg); + pParse->zErrMsg = zMsg; + pParse->rc = SQLITE_ERROR; ++ pParse->pWith = 0; + } + } + +@@ -30517,12 +31417,18 @@ + } + SQLITE_PRIVATE int sqlite3StrICmp(const char *zLeft, const char *zRight){ + unsigned char *a, *b; +- int c; ++ int c, x; + a = (unsigned char *)zLeft; + b = (unsigned char *)zRight; + for(;;){ +- c = (int)UpperToLower[*a] - (int)UpperToLower[*b]; +- if( c || *a==0 ) break; ++ c = *a; ++ x = *b; ++ if( c==x ){ ++ if( c==0 ) break; ++ }else{ ++ c = (int)UpperToLower[c] - (int)UpperToLower[x]; ++ if( c ) break; ++ } + a++; + b++; + } +@@ -30541,6 +31447,19 @@ + return N<0 ? 0 : UpperToLower[*a] - UpperToLower[*b]; + } + ++/* ++** Compute an 8-bit hash on a string that is insensitive to case differences ++*/ ++SQLITE_PRIVATE u8 sqlite3StrIHash(const char *z){ ++ u8 h = 0; ++ if( z==0 ) return 0; ++ while( z[0] ){ ++ h += UpperToLower[(unsigned char)z[0]]; ++ z++; ++ } ++ return h; ++} ++ + /* + ** Compute 10 to the E-th power. Examples: E==1 results in 10. + ** E==2 results in 100. E==50 results in 1.0e50. +@@ -30550,15 +31469,15 @@ + static LONGDOUBLE_TYPE sqlite3Pow10(int E){ + #if defined(_MSC_VER) + static const LONGDOUBLE_TYPE x[] = { +- 1.0e+001, +- 1.0e+002, +- 1.0e+004, +- 1.0e+008, +- 1.0e+016, +- 1.0e+032, +- 1.0e+064, +- 1.0e+128, +- 1.0e+256 ++ 1.0e+001L, ++ 1.0e+002L, ++ 1.0e+004L, ++ 1.0e+008L, ++ 1.0e+016L, ++ 1.0e+032L, ++ 1.0e+064L, ++ 1.0e+128L, ++ 1.0e+256L + }; + LONGDOUBLE_TYPE r = 1.0; + int i; +@@ -30588,8 +31507,15 @@ + ** uses the encoding enc. The string is not necessarily zero-terminated. + ** + ** Return TRUE if the result is a valid real number (or integer) and FALSE +-** if the string is empty or contains extraneous text. Valid numbers +-** are in one of these formats: ++** if the string is empty or contains extraneous text. More specifically ++** return ++** 1 => The input string is a pure integer ++** 2 or more => The input has a decimal point or eNNN clause ++** 0 or less => The input string is not a valid number ++** -1 => Not a valid number, but has a valid prefix which ++** includes a decimal point and/or an eNNN clause ++** ++** Valid numbers are in one of these formats: + ** + ** [+-]digits[E[+-]digits] + ** [+-]digits.[digits][E[+-]digits] +@@ -30602,10 +31528,13 @@ + ** returns FALSE but it still converts the prefix and writes the result + ** into *pResult. + */ ++#if defined(_MSC_VER) ++#pragma warning(disable : 4756) ++#endif + SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ + #ifndef SQLITE_OMIT_FLOATING_POINT + int incr; +- const char *zEnd = z + length; ++ const char *zEnd; + /* sign * significand * (10 ^ (esign * exponent)) */ + int sign = 1; /* sign of significand */ + i64 s = 0; /* significand */ +@@ -30614,20 +31543,25 @@ + int e = 0; /* exponent */ + int eValid = 1; /* True exponent is either not used or is well-formed */ + double result; +- int nDigits = 0; +- int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ ++ int nDigit = 0; /* Number of digits processed */ ++ int eType = 1; /* 1: pure integer, 2+: fractional -1 or less: bad UTF16 */ + + assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); + *pResult = 0.0; /* Default return value, in case of an error */ ++ if( length==0 ) return 0; + + if( enc==SQLITE_UTF8 ){ + incr = 1; ++ zEnd = z + length; + }else{ + int i; + incr = 2; ++ length &= ~1; + assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); ++ testcase( enc==SQLITE_UTF16LE ); ++ testcase( enc==SQLITE_UTF16BE ); + for(i=3-enc; i<length && z[i]==0; i+=2){} +- nonNum = i<length; ++ if( i<length ) eType = -100; + zEnd = &z[i^1]; + z += (enc&1); + } +@@ -30645,27 +31579,30 @@ + } + + /* copy max significant digits to significand */ +- while( z<zEnd && sqlite3Isdigit(*z) && s<((LARGEST_INT64-9)/10) ){ ++ while( z<zEnd && sqlite3Isdigit(*z) ){ + s = s*10 + (*z - '0'); +- z+=incr; nDigits++; ++ z+=incr; nDigit++; ++ if( s>=((LARGEST_INT64-9)/10) ){ ++ /* skip non-significant significand digits ++ ** (increase exponent by d to shift decimal left) */ ++ while( z<zEnd && sqlite3Isdigit(*z) ){ z+=incr; d++; } ++ } + } +- +- /* skip non-significant significand digits +- ** (increase exponent by d to shift decimal left) */ +- while( z<zEnd && sqlite3Isdigit(*z) ){ z+=incr; nDigits++; d++; } + if( z>=zEnd ) goto do_atof_calc; + + /* if decimal point is present */ + if( *z=='.' ){ + z+=incr; ++ eType++; + /* copy digits from after decimal to significand + ** (decrease exponent by d to shift decimal right) */ + while( z<zEnd && sqlite3Isdigit(*z) ){ + if( s<((LARGEST_INT64-9)/10) ){ + s = s*10 + (*z - '0'); + d--; ++ nDigit++; + } +- z+=incr; nDigits++; ++ z+=incr; + } + } + if( z>=zEnd ) goto do_atof_calc; +@@ -30674,6 +31611,7 @@ + if( *z=='e' || *z=='E' ){ + z+=incr; + eValid = 0; ++ eType++; + + /* This branch is needed to avoid a (harmless) buffer overread. The + ** special comment alerts the mutation tester that the correct answer +@@ -30772,11 +31710,20 @@ + *pResult = result; + + /* return true if number and no extra non-whitespace chracters after */ +- return z==zEnd && nDigits>0 && eValid && nonNum==0; ++ if( z==zEnd && nDigit>0 && eValid && eType>0 ){ ++ return eType; ++ }else if( eType>=2 && (eType==3 || eValid) && nDigit>0 ){ ++ return -1; ++ }else{ ++ return 0; ++ } + #else + return !sqlite3Atoi64(z, pResult, length, enc); + #endif /* SQLITE_OMIT_FLOATING_POINT */ + } ++#if defined(_MSC_VER) ++#pragma warning(default : 4756) ++#endif + + /* + ** Compare the 19-character string zNum against the text representation +@@ -30815,6 +31762,7 @@ + ** + ** Returns: + ** ++** -1 Not even a prefix of the input text looks like an integer + ** 0 Successful transformation. Fits in a 64-bit signed integer. + ** 1 Excess non-space text after the integer value + ** 2 Integer too large for a 64-bit signed integer or is malformed +@@ -30874,9 +31822,9 @@ + *pNum = (i64)u; + } + rc = 0; +- if( (i==0 && zStart==zNum) /* No digits */ +- || nonNum /* UTF16 with high-order bytes non-zero */ +- ){ ++ if( i==0 && zStart==zNum ){ /* No digits */ ++ rc = -1; ++ }else if( nonNum ){ /* UTF16 with high-order bytes non-zero */ + rc = 1; + }else if( &zNum[i]<zEnd ){ /* Extra bytes at the end */ + int jj = i; +@@ -31107,23 +32055,12 @@ + SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ + u32 a,b,s; + +- a = *p; +- /* a: p0 (unmasked) */ +- if (!(a&0x80)) +- { +- *v = a; ++ if( ((signed char*)p)[0]>=0 ){ ++ *v = *p; + return 1; + } +- +- p++; +- b = *p; +- /* b: p1 (unmasked) */ +- if (!(b&0x80)) +- { +- a &= 0x7f; +- a = a<<7; +- a |= b; +- *v = a; ++ if( ((signed char*)p)[1]>=0 ){ ++ *v = ((u32)(p[0]&0x7f)<<7) | p[1]; + return 2; + } + +@@ -31131,8 +32068,9 @@ + assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) ); + assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) ); + +- p++; +- a = a<<14; ++ a = ((u32)p[0])<<14; ++ b = p[1]; ++ p += 2; + a |= *p; + /* a: p0<<14 | p2 (unmasked) */ + if (!(a&0x80)) +@@ -31461,7 +32399,7 @@ + return (u8)(h & 0xf); + } + +-#if !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) ++#if !defined(SQLITE_OMIT_BLOB_LITERAL) + /* + ** Convert a BLOB literal of the form "x'hhhhhh'" into its binary + ** value. Return a pointer to its binary value. Space to hold the +@@ -31482,7 +32420,7 @@ + } + return zBlob; + } +-#endif /* !SQLITE_OMIT_BLOB_LITERAL || SQLITE_HAS_CODEC */ ++#endif /* !SQLITE_OMIT_BLOB_LITERAL */ + + /* + ** Log an error that is an API call on a connection pointer that should +@@ -31715,7 +32653,7 @@ + #endif /* SQLITE_OMIT_VIRTUALTABLE */ + + #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ +- defined(SQLITE_ENABLE_STAT3_OR_STAT4) || \ ++ defined(SQLITE_ENABLE_STAT4) || \ + defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) + /* + ** Convert a LogEst into an integer. +@@ -31733,7 +32671,7 @@ + defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) + if( x>60 ) return (u64)LARGEST_INT64; + #else +- /* If only SQLITE_ENABLE_STAT3_OR_STAT4 is on, then the largest input ++ /* If only SQLITE_ENABLE_STAT4 is on, then the largest input + ** possible to this routine is 310, resulting in a maximum x of 31 */ + assert( x<=60 ); + #endif +@@ -32156,30 +33094,30 @@ + /* 23 */ "SeekLE" OpHelp("key=r[P3@P4]"), + /* 24 */ "SeekGE" OpHelp("key=r[P3@P4]"), + /* 25 */ "SeekGT" OpHelp("key=r[P3@P4]"), +- /* 26 */ "IfNoHope" OpHelp("key=r[P3@P4]"), +- /* 27 */ "NoConflict" OpHelp("key=r[P3@P4]"), +- /* 28 */ "NotFound" OpHelp("key=r[P3@P4]"), +- /* 29 */ "Found" OpHelp("key=r[P3@P4]"), +- /* 30 */ "SeekRowid" OpHelp("intkey=r[P3]"), +- /* 31 */ "NotExists" OpHelp("intkey=r[P3]"), +- /* 32 */ "Last" OpHelp(""), +- /* 33 */ "IfSmaller" OpHelp(""), +- /* 34 */ "SorterSort" OpHelp(""), +- /* 35 */ "Sort" OpHelp(""), +- /* 36 */ "Rewind" OpHelp(""), +- /* 37 */ "IdxLE" OpHelp("key=r[P3@P4]"), +- /* 38 */ "IdxGT" OpHelp("key=r[P3@P4]"), +- /* 39 */ "IdxLT" OpHelp("key=r[P3@P4]"), +- /* 40 */ "IdxGE" OpHelp("key=r[P3@P4]"), +- /* 41 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), +- /* 42 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), ++ /* 26 */ "IfNotOpen" OpHelp("if( !csr[P1] ) goto P2"), ++ /* 27 */ "IfNoHope" OpHelp("key=r[P3@P4]"), ++ /* 28 */ "NoConflict" OpHelp("key=r[P3@P4]"), ++ /* 29 */ "NotFound" OpHelp("key=r[P3@P4]"), ++ /* 30 */ "Found" OpHelp("key=r[P3@P4]"), ++ /* 31 */ "SeekRowid" OpHelp("intkey=r[P3]"), ++ /* 32 */ "NotExists" OpHelp("intkey=r[P3]"), ++ /* 33 */ "Last" OpHelp(""), ++ /* 34 */ "IfSmaller" OpHelp(""), ++ /* 35 */ "SorterSort" OpHelp(""), ++ /* 36 */ "Sort" OpHelp(""), ++ /* 37 */ "Rewind" OpHelp(""), ++ /* 38 */ "IdxLE" OpHelp("key=r[P3@P4]"), ++ /* 39 */ "IdxGT" OpHelp("key=r[P3@P4]"), ++ /* 40 */ "IdxLT" OpHelp("key=r[P3@P4]"), ++ /* 41 */ "IdxGE" OpHelp("key=r[P3@P4]"), ++ /* 42 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), + /* 43 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), + /* 44 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), +- /* 45 */ "Program" OpHelp(""), +- /* 46 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), +- /* 47 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), +- /* 48 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), +- /* 49 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), ++ /* 45 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), ++ /* 46 */ "Program" OpHelp(""), ++ /* 47 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), ++ /* 48 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), ++ /* 49 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), + /* 50 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), + /* 51 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), + /* 52 */ "Ne" OpHelp("IF r[P3]!=r[P1]"), +@@ -32189,83 +33127,83 @@ + /* 56 */ "Lt" OpHelp("IF r[P3]<r[P1]"), + /* 57 */ "Ge" OpHelp("IF r[P3]>=r[P1]"), + /* 58 */ "ElseNotEq" OpHelp(""), +- /* 59 */ "IncrVacuum" OpHelp(""), +- /* 60 */ "VNext" OpHelp(""), +- /* 61 */ "Init" OpHelp("Start at P2"), +- /* 62 */ "PureFunc0" OpHelp(""), +- /* 63 */ "Function0" OpHelp("r[P3]=func(r[P2@P5])"), +- /* 64 */ "PureFunc" OpHelp(""), +- /* 65 */ "Function" OpHelp("r[P3]=func(r[P2@P5])"), +- /* 66 */ "Return" OpHelp(""), +- /* 67 */ "EndCoroutine" OpHelp(""), +- /* 68 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), +- /* 69 */ "Halt" OpHelp(""), +- /* 70 */ "Integer" OpHelp("r[P2]=P1"), +- /* 71 */ "Int64" OpHelp("r[P2]=P4"), +- /* 72 */ "String" OpHelp("r[P2]='P4' (len=P1)"), +- /* 73 */ "Null" OpHelp("r[P2..P3]=NULL"), +- /* 74 */ "SoftNull" OpHelp("r[P1]=NULL"), +- /* 75 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), +- /* 76 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), +- /* 77 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), +- /* 78 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), +- /* 79 */ "SCopy" OpHelp("r[P2]=r[P1]"), +- /* 80 */ "IntCopy" OpHelp("r[P2]=r[P1]"), +- /* 81 */ "ResultRow" OpHelp("output=r[P1@P2]"), +- /* 82 */ "CollSeq" OpHelp(""), +- /* 83 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), +- /* 84 */ "RealAffinity" OpHelp(""), +- /* 85 */ "Cast" OpHelp("affinity(r[P1])"), +- /* 86 */ "Permutation" OpHelp(""), +- /* 87 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), +- /* 88 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), +- /* 89 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), +- /* 90 */ "Column" OpHelp("r[P3]=PX"), +- /* 91 */ "Affinity" OpHelp("affinity(r[P1@P2])"), +- /* 92 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), +- /* 93 */ "Count" OpHelp("r[P2]=count()"), +- /* 94 */ "ReadCookie" OpHelp(""), +- /* 95 */ "SetCookie" OpHelp(""), +- /* 96 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), +- /* 97 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), +- /* 98 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<<r[P1]"), +- /* 99 */ "ShiftRight" OpHelp("r[P3]=r[P2]>>r[P1]"), +- /* 100 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), +- /* 101 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), +- /* 102 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), +- /* 103 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), +- /* 104 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), +- /* 105 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), +- /* 106 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), +- /* 107 */ "BitNot" OpHelp("r[P2]= ~r[P1]"), +- /* 108 */ "OpenRead" OpHelp("root=P2 iDb=P3"), +- /* 109 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), +- /* 110 */ "String8" OpHelp("r[P2]='P4'"), +- /* 111 */ "OpenDup" OpHelp(""), +- /* 112 */ "OpenAutoindex" OpHelp("nColumn=P2"), +- /* 113 */ "OpenEphemeral" OpHelp("nColumn=P2"), +- /* 114 */ "SorterOpen" OpHelp(""), +- /* 115 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), +- /* 116 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), +- /* 117 */ "Close" OpHelp(""), +- /* 118 */ "ColumnsUsed" OpHelp(""), +- /* 119 */ "SeekHit" OpHelp("seekHit=P2"), +- /* 120 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), +- /* 121 */ "NewRowid" OpHelp("r[P2]=rowid"), +- /* 122 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), +- /* 123 */ "Delete" OpHelp(""), +- /* 124 */ "ResetCount" OpHelp(""), +- /* 125 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), +- /* 126 */ "SorterData" OpHelp("r[P2]=data"), +- /* 127 */ "RowData" OpHelp("r[P2]=data"), +- /* 128 */ "Rowid" OpHelp("r[P2]=rowid"), +- /* 129 */ "NullRow" OpHelp(""), +- /* 130 */ "SeekEnd" OpHelp(""), ++ /* 59 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), ++ /* 60 */ "IncrVacuum" OpHelp(""), ++ /* 61 */ "VNext" OpHelp(""), ++ /* 62 */ "Init" OpHelp("Start at P2"), ++ /* 63 */ "PureFunc" OpHelp("r[P3]=func(r[P2@NP])"), ++ /* 64 */ "Function" OpHelp("r[P3]=func(r[P2@NP])"), ++ /* 65 */ "Return" OpHelp(""), ++ /* 66 */ "EndCoroutine" OpHelp(""), ++ /* 67 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), ++ /* 68 */ "Halt" OpHelp(""), ++ /* 69 */ "Integer" OpHelp("r[P2]=P1"), ++ /* 70 */ "Int64" OpHelp("r[P2]=P4"), ++ /* 71 */ "String" OpHelp("r[P2]='P4' (len=P1)"), ++ /* 72 */ "Null" OpHelp("r[P2..P3]=NULL"), ++ /* 73 */ "SoftNull" OpHelp("r[P1]=NULL"), ++ /* 74 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), ++ /* 75 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), ++ /* 76 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), ++ /* 77 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), ++ /* 78 */ "SCopy" OpHelp("r[P2]=r[P1]"), ++ /* 79 */ "IntCopy" OpHelp("r[P2]=r[P1]"), ++ /* 80 */ "ResultRow" OpHelp("output=r[P1@P2]"), ++ /* 81 */ "CollSeq" OpHelp(""), ++ /* 82 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), ++ /* 83 */ "RealAffinity" OpHelp(""), ++ /* 84 */ "Cast" OpHelp("affinity(r[P1])"), ++ /* 85 */ "Permutation" OpHelp(""), ++ /* 86 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), ++ /* 87 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), ++ /* 88 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), ++ /* 89 */ "Column" OpHelp("r[P3]=PX"), ++ /* 90 */ "Affinity" OpHelp("affinity(r[P1@P2])"), ++ /* 91 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), ++ /* 92 */ "Count" OpHelp("r[P2]=count()"), ++ /* 93 */ "ReadCookie" OpHelp(""), ++ /* 94 */ "SetCookie" OpHelp(""), ++ /* 95 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), ++ /* 96 */ "OpenRead" OpHelp("root=P2 iDb=P3"), ++ /* 97 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), ++ /* 98 */ "OpenDup" OpHelp(""), ++ /* 99 */ "OpenAutoindex" OpHelp("nColumn=P2"), ++ /* 100 */ "OpenEphemeral" OpHelp("nColumn=P2"), ++ /* 101 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), ++ /* 102 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), ++ /* 103 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<<r[P1]"), ++ /* 104 */ "ShiftRight" OpHelp("r[P3]=r[P2]>>r[P1]"), ++ /* 105 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), ++ /* 106 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), ++ /* 107 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), ++ /* 108 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), ++ /* 109 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), ++ /* 110 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), ++ /* 111 */ "SorterOpen" OpHelp(""), ++ /* 112 */ "BitNot" OpHelp("r[P2]= ~r[P1]"), ++ /* 113 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), ++ /* 114 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), ++ /* 115 */ "String8" OpHelp("r[P2]='P4'"), ++ /* 116 */ "Close" OpHelp(""), ++ /* 117 */ "ColumnsUsed" OpHelp(""), ++ /* 118 */ "SeekHit" OpHelp("seekHit=P2"), ++ /* 119 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), ++ /* 120 */ "NewRowid" OpHelp("r[P2]=rowid"), ++ /* 121 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), ++ /* 122 */ "Delete" OpHelp(""), ++ /* 123 */ "ResetCount" OpHelp(""), ++ /* 124 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), ++ /* 125 */ "SorterData" OpHelp("r[P2]=data"), ++ /* 126 */ "RowData" OpHelp("r[P2]=data"), ++ /* 127 */ "Rowid" OpHelp("r[P2]=rowid"), ++ /* 128 */ "NullRow" OpHelp(""), ++ /* 129 */ "SeekEnd" OpHelp(""), ++ /* 130 */ "IdxInsert" OpHelp("key=r[P2]"), + /* 131 */ "SorterInsert" OpHelp("key=r[P2]"), +- /* 132 */ "IdxInsert" OpHelp("key=r[P2]"), +- /* 133 */ "IdxDelete" OpHelp("key=r[P2@P3]"), +- /* 134 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), +- /* 135 */ "IdxRowid" OpHelp("r[P2]=rowid"), ++ /* 132 */ "IdxDelete" OpHelp("key=r[P2@P3]"), ++ /* 133 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), ++ /* 134 */ "IdxRowid" OpHelp("r[P2]=rowid"), ++ /* 135 */ "FinishSeek" OpHelp(""), + /* 136 */ "Destroy" OpHelp(""), + /* 137 */ "Clear" OpHelp(""), + /* 138 */ "ResetSorter" OpHelp(""), +@@ -32275,12 +33213,12 @@ + /* 142 */ "LoadAnalysis" OpHelp(""), + /* 143 */ "DropTable" OpHelp(""), + /* 144 */ "DropIndex" OpHelp(""), +- /* 145 */ "Real" OpHelp("r[P2]=P4"), +- /* 146 */ "DropTrigger" OpHelp(""), +- /* 147 */ "IntegrityCk" OpHelp(""), +- /* 148 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), +- /* 149 */ "Param" OpHelp(""), +- /* 150 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), ++ /* 145 */ "DropTrigger" OpHelp(""), ++ /* 146 */ "IntegrityCk" OpHelp(""), ++ /* 147 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), ++ /* 148 */ "Param" OpHelp(""), ++ /* 149 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), ++ /* 150 */ "Real" OpHelp("r[P2]=P4"), + /* 151 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 152 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), + /* 153 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), +@@ -32289,20 +33227,23 @@ + /* 156 */ "AggValue" OpHelp("r[P3]=value N=P2"), + /* 157 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 158 */ "Expire" OpHelp(""), +- /* 159 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), +- /* 160 */ "VBegin" OpHelp(""), +- /* 161 */ "VCreate" OpHelp(""), +- /* 162 */ "VDestroy" OpHelp(""), +- /* 163 */ "VOpen" OpHelp(""), +- /* 164 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), +- /* 165 */ "VRename" OpHelp(""), +- /* 166 */ "Pagecount" OpHelp(""), +- /* 167 */ "MaxPgcnt" OpHelp(""), +- /* 168 */ "Trace" OpHelp(""), +- /* 169 */ "CursorHint" OpHelp(""), +- /* 170 */ "Noop" OpHelp(""), +- /* 171 */ "Explain" OpHelp(""), +- /* 172 */ "Abortable" OpHelp(""), ++ /* 159 */ "CursorLock" OpHelp(""), ++ /* 160 */ "CursorUnlock" OpHelp(""), ++ /* 161 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), ++ /* 162 */ "VBegin" OpHelp(""), ++ /* 163 */ "VCreate" OpHelp(""), ++ /* 164 */ "VDestroy" OpHelp(""), ++ /* 165 */ "VOpen" OpHelp(""), ++ /* 166 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), ++ /* 167 */ "VRename" OpHelp(""), ++ /* 168 */ "Pagecount" OpHelp(""), ++ /* 169 */ "MaxPgcnt" OpHelp(""), ++ /* 170 */ "Trace" OpHelp(""), ++ /* 171 */ "CursorHint" OpHelp(""), ++ /* 172 */ "ReleaseReg" OpHelp("release r[P1@P2] mask P3"), ++ /* 173 */ "Noop" OpHelp(""), ++ /* 174 */ "Explain" OpHelp(""), ++ /* 175 */ "Abortable" OpHelp(""), + }; + return azName[i]; + } +@@ -32417,13 +33358,29 @@ + # include <sys/param.h> + #endif /* SQLITE_ENABLE_LOCKING_STYLE */ + +-#if defined(__APPLE__) && ((__MAC_OS_X_VERSION_MIN_REQUIRED > 1050) || \ +- (__IPHONE_OS_VERSION_MIN_REQUIRED > 2000)) +-# if (!defined(TARGET_OS_EMBEDDED) || (TARGET_OS_EMBEDDED==0)) \ +- && (!defined(TARGET_IPHONE_SIMULATOR) || (TARGET_IPHONE_SIMULATOR==0)) +-# define HAVE_GETHOSTUUID 1 +-# else +-# warning "gethostuuid() is disabled." ++/* ++** Try to determine if gethostuuid() is available based on standard ++** macros. This might sometimes compute the wrong value for some ++** obscure platforms. For those cases, simply compile with one of ++** the following: ++** ++** -DHAVE_GETHOSTUUID=0 ++** -DHAVE_GETHOSTUUID=1 ++** ++** None if this matters except when building on Apple products with ++** -DSQLITE_ENABLE_LOCKING_STYLE. ++*/ ++#ifndef HAVE_GETHOSTUUID ++# define HAVE_GETHOSTUUID 0 ++# if defined(__APPLE__) && ((__MAC_OS_X_VERSION_MIN_REQUIRED > 1050) || \ ++ (__IPHONE_OS_VERSION_MIN_REQUIRED > 2000)) ++# if (!defined(TARGET_OS_EMBEDDED) || (TARGET_OS_EMBEDDED==0)) \ ++ && (!defined(TARGET_IPHONE_SIMULATOR) || (TARGET_IPHONE_SIMULATOR==0)) ++# undef HAVE_GETHOSTUUID ++# define HAVE_GETHOSTUUID 1 ++# else ++# warning "gethostuuid() is disabled." ++# endif + # endif + #endif + +@@ -32650,7 +33607,7 @@ + ****************************************************************************** + ** + ** This file contains inline asm code for retrieving "high-performance" +-** counters for x86 class CPUs. ++** counters for x86 and x86_64 class CPUs. + */ + #ifndef SQLITE_HWTIME_H + #define SQLITE_HWTIME_H +@@ -32661,8 +33618,9 @@ + ** processor and returns that value. This can be used for high-res + ** profiling. + */ +-#if (defined(__GNUC__) || defined(_MSC_VER)) && \ +- (defined(i386) || defined(__i386__) || defined(_M_IX86)) ++#if !defined(__STRICT_ANSI__) && \ ++ (defined(__GNUC__) || defined(_MSC_VER)) && \ ++ (defined(i386) || defined(__i386__) || defined(_M_IX86)) + + #if defined(__GNUC__) + +@@ -32683,7 +33641,7 @@ + + #endif + +-#elif (defined(__GNUC__) && defined(__x86_64__)) ++#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long val; +@@ -32691,7 +33649,7 @@ + return val; + } + +-#elif (defined(__GNUC__) && defined(__ppc__)) ++#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long long retval; +@@ -32708,14 +33666,13 @@ + + #else + +- #error Need implementation of sqlite3Hwtime() for your platform. +- + /* +- ** To compile without implementing sqlite3Hwtime() for your platform, +- ** you can remove the above #error and use the following +- ** stub function. You will lose timing support for many +- ** of the debugging and testing utilities, but it should at +- ** least compile and run. ++ ** asm() is needed for hardware timing support. Without asm(), ++ ** disable the sqlite3Hwtime() routine. ++ ** ++ ** sqlite3Hwtime() is only used for some obscure debugging ++ ** and analysis configurations, not in any deliverable, so this ++ ** should not be a great loss. + */ + SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } + +@@ -33031,13 +33988,14 @@ + #if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) + # ifdef __ANDROID__ + { "ioctl", (sqlite3_syscall_ptr)(int(*)(int, int, ...))ioctl, 0 }, ++#define osIoctl ((int(*)(int,int,...))aSyscall[28].pCurrent) + # else + { "ioctl", (sqlite3_syscall_ptr)ioctl, 0 }, ++#define osIoctl ((int(*)(int,unsigned long,...))aSyscall[28].pCurrent) + # endif + #else + { "ioctl", (sqlite3_syscall_ptr)0, 0 }, + #endif +-#define osIoctl ((int(*)(int,int,...))aSyscall[28].pCurrent) + + }; /* End of the overrideable system calls */ + +@@ -33182,7 +34140,7 @@ + sqlite3_log(SQLITE_WARNING, + "attempt to open \"%s\" as file descriptor %d", z, fd); + fd = -1; +- if( osOpen("/dev/null", f, m)<0 ) break; ++ if( osOpen("/dev/null", O_RDONLY, m)<0 ) break; + } + if( fd>=0 ){ + if( m!=0 ){ +@@ -34058,8 +35016,9 @@ + struct flock *pLock, /* The description of the lock */ + unixFile *pFile /* Structure holding timeout value */ + ){ ++ int tm = pFile->iBusyTimeout; + int rc = osFcntl(h,F_SETLK,pLock); +- while( rc<0 && pFile->iBusyTimeout>0 ){ ++ while( rc<0 && tm>0 ){ + /* On systems that support some kind of blocking file lock with a timeout, + ** make appropriate changes here to invoke that blocking file lock. On + ** generic posix, however, there is no such API. So we simply try the +@@ -34067,7 +35026,7 @@ + ** the lock is obtained. */ + usleep(1000); + rc = osFcntl(h,F_SETLK,pLock); +- pFile->iBusyTimeout--; ++ tm--; + } + return rc; + } +@@ -36488,7 +37447,9 @@ + } + #ifdef SQLITE_ENABLE_SETLK_TIMEOUT + case SQLITE_FCNTL_LOCK_TIMEOUT: { ++ int iOld = pFile->iBusyTimeout; + pFile->iBusyTimeout = *(int*)pArg; ++ *(int*)pArg = iOld; + return SQLITE_OK; + } + #endif +@@ -36807,13 +37768,20 @@ + assert( n>=1 && n<=SQLITE_SHM_NLOCK ); + + if( pShmNode->hShm>=0 ){ ++ int res; + /* Initialize the locking parameters */ + f.l_type = lockType; + f.l_whence = SEEK_SET; + f.l_start = ofst; + f.l_len = n; +- rc = osSetPosixAdvisoryLock(pShmNode->hShm, &f, pFile); +- rc = (rc!=(-1)) ? SQLITE_OK : SQLITE_BUSY; ++ res = osSetPosixAdvisoryLock(pShmNode->hShm, &f, pFile); ++ if( res==-1 ){ ++#ifdef SQLITE_ENABLE_SETLK_TIMEOUT ++ rc = (pFile->iBusyTimeout ? SQLITE_BUSY_TIMEOUT : SQLITE_BUSY); ++#else ++ rc = SQLITE_BUSY; ++#endif ++ } + } + + /* Update the global lock state and do debug tracing */ +@@ -37069,10 +38037,12 @@ + + if( pInode->bProcessLock==0 ){ + if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ +- pShmNode->hShm = robust_open(zShm, O_RDWR|O_CREAT,(sStat.st_mode&0777)); ++ pShmNode->hShm = robust_open(zShm, O_RDWR|O_CREAT|O_NOFOLLOW, ++ (sStat.st_mode&0777)); + } + if( pShmNode->hShm<0 ){ +- pShmNode->hShm = robust_open(zShm, O_RDONLY, (sStat.st_mode&0777)); ++ pShmNode->hShm = robust_open(zShm, O_RDONLY|O_NOFOLLOW, ++ (sStat.st_mode&0777)); + if( pShmNode->hShm<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShm); + goto shm_open_err; +@@ -37308,6 +38278,25 @@ + assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 ); + assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 ); + ++ /* Check that, if this to be a blocking lock, no locks that occur later ++ ** in the following list than the lock being obtained are already held: ++ ** ++ ** 1. Checkpointer lock (ofst==1). ++ ** 2. Write lock (ofst==0). ++ ** 3. Read locks (ofst>=3 && ofst<SQLITE_SHM_NLOCK). ++ ** ++ ** In other words, if this is a blocking lock, none of the locks that ++ ** occur later in the above list than the lock being obtained may be ++ ** held. */ ++#ifdef SQLITE_ENABLE_SETLK_TIMEOUT ++ assert( (flags & SQLITE_SHM_UNLOCK) || pDbFd->iBusyTimeout==0 || ( ++ (ofst!=2) /* not RECOVER */ ++ && (ofst!=1 || (p->exclMask|p->sharedMask)==0) ++ && (ofst!=0 || (p->exclMask|p->sharedMask)<3) ++ && (ofst<3 || (p->exclMask|p->sharedMask)<(1<<ofst)) ++ )); ++#endif ++ + mask = (1<<(ofst+n)) - (1<<ofst); + assert( n>1 || mask==(1<<ofst) ); + sqlite3_mutex_enter(pShmNode->pShmMutex); +@@ -38279,6 +39268,7 @@ + UnixUnusedFd **pp; + assert( sqlite3_mutex_notheld(pInode->pLockMutex) ); + sqlite3_mutex_enter(pInode->pLockMutex); ++ flags &= (SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE); + for(pp=&pInode->pUnused; *pp && (*pp)->flags!=flags; pp=&((*pp)->pNext)); + pUnused = *pp; + if( pUnused ){ +@@ -38332,7 +39322,7 @@ + ** If the SQLITE_ENABLE_8_3_NAMES option is enabled, then the + ** original filename is unavailable. But 8_3_NAMES is only used for + ** FAT filesystems and permissions do not matter there, so just use +-** the default permissions. ++** the default permissions. In 8_3_NAMES mode, leave *pMode set to zero. + */ + static int findCreateFileMode( + const char *zPath, /* Path of file (possibly) being created */ +@@ -38421,7 +39411,7 @@ + unixFile *p = (unixFile *)pFile; + int fd = -1; /* File descriptor returned by open() */ + int openFlags = 0; /* Flags to pass to open() */ +- int eType = flags&0xFFFFFF00; /* Type of file to open */ ++ int eType = flags&0x0FFF00; /* Type of file to open */ + int noLock; /* True to omit locking primitives */ + int rc = SQLITE_OK; /* Function Return Code */ + int ctrlFlags = 0; /* UNIXFILE_* flags */ +@@ -38531,7 +39521,7 @@ + if( isReadWrite ) openFlags |= O_RDWR; + if( isCreate ) openFlags |= O_CREAT; + if( isExclusive ) openFlags |= (O_EXCL|O_NOFOLLOW); +- openFlags |= (O_LARGEFILE|O_BINARY); ++ openFlags |= (O_LARGEFILE|O_BINARY|O_NOFOLLOW); + + if( fd<0 ){ + mode_t openMode; /* Permissions to create file with */ +@@ -38567,11 +39557,19 @@ + goto open_finished; + } + +- /* If this process is running as root and if creating a new rollback +- ** journal or WAL file, set the ownership of the journal or WAL to be +- ** the same as the original database. ++ /* The owner of the rollback journal or WAL file should always be the ++ ** same as the owner of the database file. Try to ensure that this is ++ ** the case. The chown() system call will be a no-op if the current ++ ** process lacks root privileges, be we should at least try. Without ++ ** this step, if a root process opens a database file, it can leave ++ ** behinds a journal/WAL that is owned by root and hence make the ++ ** database inaccessible to unprivileged processes. ++ ** ++ ** If openMode==0, then that means uid and gid are not set correctly ++ ** (probably because SQLite is configured to use 8+3 filename mode) and ++ ** in that case we do not want to attempt the chown(). + */ +- if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ ++ if( openMode && (flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL))!=0 ){ + robustFchown(fd, uid, gid); + } + } +@@ -38582,7 +39580,8 @@ + + if( p->pPreallocatedUnused ){ + p->pPreallocatedUnused->fd = fd; +- p->pPreallocatedUnused->flags = flags; ++ p->pPreallocatedUnused->flags = ++ flags & (SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE); + } + + if( isDelete ){ +@@ -38740,7 +39739,8 @@ + + if( flags==SQLITE_ACCESS_EXISTS ){ + struct stat buf; +- *pResOut = (0==osStat(zPath, &buf) && buf.st_size>0); ++ *pResOut = 0==osStat(zPath, &buf) && ++ (!S_ISREG(buf.st_mode) || buf.st_size>0); + }else{ + *pResOut = osAccess(zPath, W_OK|R_OK)==0; + } +@@ -38794,7 +39794,7 @@ + #else + int rc = SQLITE_OK; + int nByte; +- int nLink = 1; /* Number of symbolic links followed so far */ ++ int nLink = 0; /* Number of symbolic links followed so far */ + const char *zIn = zPath; /* Input path for each iteration of loop */ + char *zDel = 0; + +@@ -38823,10 +39823,11 @@ + } + + if( bLink ){ ++ nLink++; + if( zDel==0 ){ + zDel = sqlite3_malloc(nOut); + if( zDel==0 ) rc = SQLITE_NOMEM_BKPT; +- }else if( ++nLink>SQLITE_MAX_SYMLINKS ){ ++ }else if( nLink>=SQLITE_MAX_SYMLINKS ){ + rc = SQLITE_CANTOPEN_BKPT; + } + +@@ -38862,6 +39863,7 @@ + }while( rc==SQLITE_OK ); + + sqlite3_free(zDel); ++ if( rc==SQLITE_OK && nLink ) rc = SQLITE_OK_SYMLINK; + return rc; + #endif /* HAVE_READLINK && HAVE_LSTAT */ + } +@@ -39347,7 +40349,7 @@ + int fd = -1; + unixFile *pNew; + int rc = SQLITE_OK; +- int openFlags = O_RDWR | O_CREAT; ++ int openFlags = O_RDWR | O_CREAT | O_NOFOLLOW; + sqlite3_vfs dummyVfs; + int terrno = 0; + UnixUnusedFd *pUnused = NULL; +@@ -39377,7 +40379,7 @@ + } + } + if( fd<0 ){ +- openFlags = O_RDONLY; ++ openFlags = O_RDONLY | O_NOFOLLOW; + fd = robust_open(path, openFlags, 0); + terrno = errno; + } +@@ -39428,7 +40430,7 @@ + + #define PROXY_HOSTIDLEN 16 /* conch file host id length */ + +-#ifdef HAVE_GETHOSTUUID ++#if HAVE_GETHOSTUUID + /* Not always defined in the headers as it ought to be */ + extern int gethostuuid(uuid_t id, const struct timespec *wait); + #endif +@@ -39439,7 +40441,7 @@ + static int proxyGetHostID(unsigned char *pHostID, int *pError){ + assert(PROXY_HOSTIDLEN == sizeof(uuid_t)); + memset(pHostID, 0, PROXY_HOSTIDLEN); +-#ifdef HAVE_GETHOSTUUID ++#if HAVE_GETHOSTUUID + { + struct timespec timeout = {1, 0}; /* 1 sec timeout */ + if( gethostuuid(pHostID, &timeout) ){ +@@ -39503,7 +40505,7 @@ + goto end_breaklock; + } + /* write it out to the temporary break file */ +- fd = robust_open(tPath, (O_RDWR|O_CREAT|O_EXCL), 0); ++ fd = robust_open(tPath, (O_RDWR|O_CREAT|O_EXCL|O_NOFOLLOW), 0); + if( fd<0 ){ + sqlite3_snprintf(sizeof(errmsg), errmsg, "create failed (%d)", errno); + goto end_breaklock; +@@ -40113,7 +41115,7 @@ + assert( 0 ); /* The call assures that only valid opcodes are sent */ + } + } +- /*NOTREACHED*/ ++ /*NOTREACHED*/ assert(0); + return SQLITE_ERROR; + } + +@@ -40461,7 +41463,7 @@ + ****************************************************************************** + ** + ** This file contains inline asm code for retrieving "high-performance" +-** counters for x86 class CPUs. ++** counters for x86 and x86_64 class CPUs. + */ + #ifndef SQLITE_HWTIME_H + #define SQLITE_HWTIME_H +@@ -40472,8 +41474,9 @@ + ** processor and returns that value. This can be used for high-res + ** profiling. + */ +-#if (defined(__GNUC__) || defined(_MSC_VER)) && \ +- (defined(i386) || defined(__i386__) || defined(_M_IX86)) ++#if !defined(__STRICT_ANSI__) && \ ++ (defined(__GNUC__) || defined(_MSC_VER)) && \ ++ (defined(i386) || defined(__i386__) || defined(_M_IX86)) + + #if defined(__GNUC__) + +@@ -40494,7 +41497,7 @@ + + #endif + +-#elif (defined(__GNUC__) && defined(__x86_64__)) ++#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long val; +@@ -40502,7 +41505,7 @@ + return val; + } + +-#elif (defined(__GNUC__) && defined(__ppc__)) ++#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long long retval; +@@ -40519,14 +41522,13 @@ + + #else + +- #error Need implementation of sqlite3Hwtime() for your platform. +- + /* +- ** To compile without implementing sqlite3Hwtime() for your platform, +- ** you can remove the above #error and use the following +- ** stub function. You will lose timing support for many +- ** of the debugging and testing utilities, but it should at +- ** least compile and run. ++ ** asm() is needed for hardware timing support. Without asm(), ++ ** disable the sqlite3Hwtime() routine. ++ ** ++ ** sqlite3Hwtime() is only used for some obscure debugging ++ ** and analysis configurations, not in any deliverable, so this ++ ** should not be a great loss. + */ + SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } + +@@ -44085,6 +45087,7 @@ + /* Forward references to VFS helper methods used for temporary files */ + static int winGetTempname(sqlite3_vfs *, char **); + static int winIsDir(const void *); ++static BOOL winIsLongPathPrefix(const char *); + static BOOL winIsDriveLetterAndColon(const char *); + + /* +@@ -44798,6 +45801,7 @@ + rc = winOpenSharedMemory(pDbFd); + if( rc!=SQLITE_OK ) return rc; + pShm = pDbFd->pShm; ++ assert( pShm!=0 ); + } + pShmNode = pShm->pShmNode; + +@@ -45100,6 +46104,7 @@ + } + } + if( pFd->mmapSize >= iOff+nAmt ){ ++ assert( pFd->pMapRegion!=0 ); + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } +@@ -45852,7 +46857,9 @@ + if( isReadonly ){ + pFile->ctrlFlags |= WINFILE_RDONLY; + } +- if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ ++ if( (flags & SQLITE_OPEN_MAIN_DB) ++ && sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ++ ){ + pFile->ctrlFlags |= WINFILE_PSOW; + } + pFile->lastErrno = NO_ERROR; +@@ -46062,6 +47069,17 @@ + return SQLITE_OK; + } + ++/* ++** Returns non-zero if the specified path name starts with the "long path" ++** prefix. ++*/ ++static BOOL winIsLongPathPrefix( ++ const char *zPathname ++){ ++ return ( zPathname[0]=='\\' && zPathname[1]=='\\' ++ && zPathname[2]=='?' && zPathname[3]=='\\' ); ++} ++ + /* + ** Returns non-zero if the specified path name starts with a drive letter + ** followed by a colon character. +@@ -46126,10 +47144,11 @@ + char *zOut; + #endif + +- /* If this path name begins with "/X:", where "X" is any alphabetic +- ** character, discard the initial "/" from the pathname. ++ /* If this path name begins with "/X:" or "\\?\", where "X" is any ++ ** alphabetic character, discard the initial "/" from the pathname. + */ +- if( zRelative[0]=='/' && winIsDriveLetterAndColon(zRelative+1) ){ ++ if( zRelative[0]=='/' && (winIsDriveLetterAndColon(zRelative+1) ++ || winIsLongPathPrefix(zRelative+1)) ){ + zRelative++; + } + +@@ -46885,7 +47904,7 @@ + } + newSz *= 2; + if( newSz>p->szMax ) newSz = p->szMax; +- pNew = sqlite3_realloc64(p->aData, newSz); ++ pNew = sqlite3Realloc(p->aData, newSz); + if( pNew==0 ) return SQLITE_NOMEM; + p->aData = pNew; + p->szAlloc = newSz; +@@ -47332,10 +48351,11 @@ + sqlite3_vfs *pLower = sqlite3_vfs_find(0); + int sz = pLower->szOsFile; + memdb_vfs.pAppData = pLower; +- /* In all known configurations of SQLite, the size of a default +- ** sqlite3_file is greater than the size of a memdb sqlite3_file. +- ** Should that ever change, remove the following NEVER() */ +- if( NEVER(sz<sizeof(MemFile)) ) sz = sizeof(MemFile); ++ /* The following conditional can only be true when compiled for ++ ** Windows x86 and SQLITE_MAX_MMAP_SIZE=0. We always leave ++ ** it in, to be safe, but it is marked as NO_TEST since there ++ ** is no way to reach it under most builds. */ ++ if( sz<sizeof(MemFile) ) sz = sizeof(MemFile); /*NO_TEST*/ + memdb_vfs.szOsFile = sz; + return sqlite3_vfs_register(&memdb_vfs, 0); + } +@@ -48002,9 +49022,10 @@ + ** suggested cache size is set to N. */ + return p->szCache; + }else{ +- /* IMPLEMENTATION-OF: R-61436-13639 If the argument N is negative, then +- ** the number of cache pages is adjusted to use approximately abs(N*1024) +- ** bytes of memory. */ ++ /* IMPLEMANTATION-OF: R-59858-46238 If the argument N is negative, then the ++ ** number of cache pages is adjusted to be a number of pages that would ++ ** use approximately abs(N*1024) bytes of memory based on the current ++ ** page size. */ + return (int)((-1024*(i64)p->szCache)/(p->szPage+p->szExtra)); + } + } +@@ -48020,6 +49041,7 @@ + ** built-in default page cache is used instead of the application defined + ** page cache. */ + sqlite3PCacheSetDefault(); ++ assert( sqlite3GlobalConfig.pcache2.xInit!=0 ); + } + return sqlite3GlobalConfig.pcache2.xInit(sqlite3GlobalConfig.pcache2.pArg); + } +@@ -49066,6 +50088,7 @@ + + assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); + if( pCache->pFree || (pCache->nPage==0 && pcache1InitBulk(pCache)) ){ ++ assert( pCache->pFree!=0 ); + p = pCache->pFree; + pCache->pFree = p->pNext; + p->pNext = 0; +@@ -49089,13 +50112,15 @@ + } + #else + pPg = pcache1Alloc(pCache->szAlloc); +- p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; + #endif + if( benignMalloc ){ sqlite3EndBenignMalloc(); } + #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + pcache1EnterMutex(pCache->pGroup); + #endif + if( pPg==0 ) return 0; ++#ifndef SQLITE_PCACHE_SEPARATE_HEADER ++ p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; ++#endif + p->page.pBuf = pPg; + p->page.pExtra = &p[1]; + p->isBulkLocal = 0; +@@ -49420,6 +50445,7 @@ + }else{ + pGroup = &pcache1.grp; + } ++ pcache1EnterMutex(pGroup); + if( pGroup->lru.isAnchor==0 ){ + pGroup->lru.isAnchor = 1; + pGroup->lru.pLruPrev = pGroup->lru.pLruNext = &pGroup->lru; +@@ -49429,7 +50455,6 @@ + pCache->szExtra = szExtra; + pCache->szAlloc = szPage + szExtra + ROUND8(sizeof(PgHdr1)); + pCache->bPurgeable = (bPurgeable ? 1 : 0); +- pcache1EnterMutex(pGroup); + pcache1ResizeHash(pCache); + if( bPurgeable ){ + pCache->nMin = 10; +@@ -50096,7 +51121,7 @@ + /* + ** Allocate a new RowSetEntry object that is associated with the + ** given RowSet. Return a pointer to the new and completely uninitialized +-** objected. ++** object. + ** + ** In an OOM situation, the RowSet.db->mallocFailed flag is set and this + ** routine returns NULL. +@@ -50372,7 +51397,7 @@ + if( p ){ + struct RowSetEntry **ppPrevTree = &pRowSet->pForest; + if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ +- /* Only sort the current set of entiries if they need it */ ++ /* Only sort the current set of entries if they need it */ + p = rowSetEntrySort(p); + } + for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ +@@ -50594,6 +51619,11 @@ + /* Return the sqlite3_file object for the WAL file */ + SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal); + ++#ifdef SQLITE_ENABLE_SETLK_TIMEOUT ++SQLITE_PRIVATE int sqlite3WalWriteLock(Wal *pWal, int bLock); ++SQLITE_PRIVATE void sqlite3WalDb(Wal *pWal, sqlite3 *db); ++#endif ++ + #endif /* ifndef SQLITE_OMIT_WAL */ + #endif /* SQLITE_WAL_H */ + +@@ -50984,20 +52014,6 @@ + */ + #define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1) + +-/* +-** A macro used for invoking the codec if there is one +-*/ +-#ifdef SQLITE_HAS_CODEC +-# define CODEC1(P,D,N,X,E) \ +- if( P->xCodec && P->xCodec(P->pCodec,D,N,X)==0 ){ E; } +-# define CODEC2(P,D,N,X,E,O) \ +- if( P->xCodec==0 ){ O=(char*)D; }else \ +- if( (O=(char*)(P->xCodec(P->pCodec,D,N,X)))==0 ){ E; } +-#else +-# define CODEC1(P,D,N,X,E) /* NO-OP */ +-# define CODEC2(P,D,N,X,E,O) O=(char*)D +-#endif +- + /* + ** The maximum allowed sector size. 64KiB. If the xSectorsize() method + ** returns a value larger than this, then MAX_SECTOR_SIZE is used instead. +@@ -51283,12 +52299,6 @@ + #endif + void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */ + int (*xGet)(Pager*,Pgno,DbPage**,int); /* Routine to fetch a patch */ +-#ifdef SQLITE_HAS_CODEC +- void *(*xCodec)(void*,void*,Pgno,int); /* Routine for en/decoding data */ +- void (*xCodecSizeChng)(void*,int,int); /* Notify of page size changes */ +- void (*xCodecFree)(void*); /* Destructor for the codec */ +- void *pCodec; /* First argument to xCodec... methods */ +-#endif + char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ + PCache *pPCache; /* Pointer to page cache object */ + #ifndef SQLITE_OMIT_WAL +@@ -51415,9 +52425,6 @@ + SQLITE_PRIVATE int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){ + if( pPager->fd->pMethods==0 ) return 0; + if( sqlite3PCacheIsDirty(pPager->pPCache) ) return 0; +-#ifdef SQLITE_HAS_CODEC +- if( pPager->xCodec!=0 ) return 0; +-#endif + #ifndef SQLITE_OMIT_WAL + if( pPager->pWal ){ + u32 iRead = 0; +@@ -51651,11 +52658,7 @@ + if( pPager->errCode ){ + pPager->xGet = getPageError; + #if SQLITE_MAX_MMAP_SIZE>0 +- }else if( USEFETCH(pPager) +-#ifdef SQLITE_HAS_CODEC +- && pPager->xCodec==0 +-#endif +- ){ ++ }else if( USEFETCH(pPager) ){ + pPager->xGet = getPageMMap; + #endif /* SQLITE_MAX_MMAP_SIZE>0 */ + }else{ +@@ -51750,6 +52753,7 @@ + } + IOTRACE(("UNLOCK %p %d\n", pPager, eLock)) + } ++ pPager->changeCountDone = pPager->tempFile; /* ticket fb3b3024ea238d5c */ + return rc; + } + +@@ -51937,6 +52941,7 @@ + len = 0; + } + zMaster[len] = '\0'; ++ zMaster[len+1] = '\0'; + + return SQLITE_OK; + } +@@ -52470,7 +53475,6 @@ + ** code is cleared and the cache reset in the block below. + */ + assert( pPager->errCode || pPager->eState!=PAGER_ERROR ); +- pPager->changeCountDone = 0; + pPager->eState = PAGER_OPEN; + } + +@@ -52734,7 +53738,6 @@ + && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) + ){ + rc2 = pagerUnlockDb(pPager, SHARED_LOCK); +- pPager->changeCountDone = 0; + } + pPager->eState = PAGER_READER; + pPager->setMaster = 0; +@@ -52803,35 +53806,6 @@ + return cksum; + } + +-/* +-** Report the current page size and number of reserved bytes back +-** to the codec. +-*/ +-#ifdef SQLITE_HAS_CODEC +-static void pagerReportSize(Pager *pPager){ +- if( pPager->xCodecSizeChng ){ +- pPager->xCodecSizeChng(pPager->pCodec, pPager->pageSize, +- (int)pPager->nReserve); +- } +-} +-#else +-# define pagerReportSize(X) /* No-op if we do not support a codec */ +-#endif +- +-#ifdef SQLITE_HAS_CODEC +-/* +-** Make sure the number of reserved bits is the same in the destination +-** pager as it is in the source. This comes up when a VACUUM changes the +-** number of reserved bits to the "optimal" amount. +-*/ +-SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager *pDest, Pager *pSrc){ +- if( pDest->nReserve!=pSrc->nReserve ){ +- pDest->nReserve = pSrc->nReserve; +- pagerReportSize(pDest); +- } +-} +-#endif +- + /* + ** Read a single page from either the journal file (if isMainJrnl==1) or + ** from the sub-journal (if isMainJrnl==0) and playback that page. +@@ -52883,11 +53857,6 @@ + char *aData; /* Temporary storage for the page */ + sqlite3_file *jfd; /* The file descriptor for the journal file */ + int isSynced; /* True if journal page is synced */ +-#ifdef SQLITE_HAS_CODEC +- /* The jrnlEnc flag is true if Journal pages should be passed through +- ** the codec. It is false for pure in-memory journals. */ +- const int jrnlEnc = (isMainJrnl || pPager->subjInMemory==0); +-#endif + + assert( (isMainJrnl&~1)==0 ); /* isMainJrnl is 0 or 1 */ + assert( (isSavepnt&~1)==0 ); /* isSavepnt is 0 or 1 */ +@@ -52950,7 +53919,6 @@ + */ + if( pgno==1 && pPager->nReserve!=((u8*)aData)[20] ){ + pPager->nReserve = ((u8*)aData)[20]; +- pagerReportSize(pPager); + } + + /* If the pager is in CACHEMOD state, then there must be a copy of this +@@ -53018,26 +53986,12 @@ + ** is if the data was just read from an in-memory sub-journal. In that + ** case it must be encrypted here before it is copied into the database + ** file. */ +-#ifdef SQLITE_HAS_CODEC +- if( !jrnlEnc ){ +- CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM_BKPT, aData); +- rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); +- CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM_BKPT); +- }else +-#endif + rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); + + if( pgno>pPager->dbFileSize ){ + pPager->dbFileSize = pgno; + } + if( pPager->pBackup ){ +-#ifdef SQLITE_HAS_CODEC +- if( jrnlEnc ){ +- CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM_BKPT); +- sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); +- CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM_BKPT,aData); +- }else +-#endif + sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); + } + }else if( !isMainJrnl && pPg==0 ){ +@@ -53088,11 +54042,6 @@ + if( pgno==1 ){ + memcpy(&pPager->dbFileVers, &((u8*)pData)[24],sizeof(pPager->dbFileVers)); + } +- +- /* Decode the page just read from disk */ +-#if SQLITE_HAS_CODEC +- if( jrnlEnc ){ CODEC1(pPager, pData, pPg->pgno, 3, rc=SQLITE_NOMEM_BKPT); } +-#endif + sqlite3PcacheRelease(pPg); + } + return rc; +@@ -53173,15 +54122,16 @@ + rc = sqlite3OsFileSize(pMaster, &nMasterJournal); + if( rc!=SQLITE_OK ) goto delmaster_out; + nMasterPtr = pVfs->mxPathname+1; +- zMasterJournal = sqlite3Malloc(nMasterJournal + nMasterPtr + 1); ++ zMasterJournal = sqlite3Malloc(nMasterJournal + nMasterPtr + 2); + if( !zMasterJournal ){ + rc = SQLITE_NOMEM_BKPT; + goto delmaster_out; + } +- zMasterPtr = &zMasterJournal[nMasterJournal+1]; ++ zMasterPtr = &zMasterJournal[nMasterJournal+2]; + rc = sqlite3OsRead(pMaster, zMasterJournal, (int)nMasterJournal, 0); + if( rc!=SQLITE_OK ) goto delmaster_out; + zMasterJournal[nMasterJournal] = 0; ++ zMasterJournal[nMasterJournal+1] = 0; + + zJournal = zMasterJournal; + while( (zJournal-zMasterJournal)<nMasterJournal ){ +@@ -53194,9 +54144,12 @@ + /* One of the journals pointed to by the master journal exists. + ** Open it and check if it points at the master journal. If + ** so, return without deleting the master journal file. ++ ** NB: zJournal is really a MAIN_JOURNAL. But call it a ++ ** MASTER_JOURNAL here so that the VFS will not send the zJournal ++ ** name into sqlite3_database_file_object(). + */ + int c; +- int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MAIN_JOURNAL); ++ int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MASTER_JOURNAL); + rc = sqlite3OsOpen(pVfs, zJournal, pJournal, flags, 0); + if( rc!=SQLITE_OK ){ + goto delmaster_out; +@@ -53651,8 +54604,6 @@ + memcpy(&pPager->dbFileVers, dbFileVers, sizeof(pPager->dbFileVers)); + } + } +- CODEC1(pPager, pPg->pData, pPg->pgno, 3, rc = SQLITE_NOMEM_BKPT); +- + PAGER_INCR(sqlite3_pager_readdb_count); + PAGER_INCR(pPager->nRead); + IOTRACE(("PGIN %p %d\n", pPager, pPg->pgno)); +@@ -54396,7 +55347,6 @@ + if( nReserve<0 ) nReserve = pPager->nReserve; + assert( nReserve>=0 && nReserve<1000 ); + pPager->nReserve = (i16)nReserve; +- pagerReportSize(pPager); + pagerFixMaplimit(pPager); + } + return rc; +@@ -54792,11 +55742,6 @@ + sqlite3OsClose(pPager->fd); + sqlite3PageFree(pTmp); + sqlite3PcacheClose(pPager->pPCache); +- +-#ifdef SQLITE_HAS_CODEC +- if( pPager->xCodecFree ) pPager->xCodecFree(pPager->pCodec); +-#endif +- + assert( !pPager->aSavepoint && !pPager->pInJournal ); + assert( !isOpen(pPager->jfd) && !isOpen(pPager->sjfd) ); + +@@ -55047,8 +55992,7 @@ + assert( (pList->flags&PGHDR_NEED_SYNC)==0 ); + if( pList->pgno==1 ) pager_write_changecounter(pList); + +- /* Encode the database */ +- CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM_BKPT, pData); ++ pData = pList->pData; + + /* Write out the page data. */ + rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); +@@ -55137,12 +56081,6 @@ + void *pData = pPg->pData; + i64 offset = (i64)pPager->nSubRec*(4+pPager->pageSize); + char *pData2; +- +-#if SQLITE_HAS_CODEC +- if( !pPager->subjInMemory ){ +- CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM_BKPT, pData2); +- }else +-#endif + pData2 = pData; + PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno)); + rc = write32bits(pPager->sjfd, offset, pPg->pgno); +@@ -55338,7 +56276,8 @@ + int pcacheSize = sqlite3PcacheSize(); /* Bytes to allocate for PCache */ + u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ + const char *zUri = 0; /* URI args to copy */ +- int nUri = 0; /* Number of bytes of URI args at *zUri */ ++ int nUriByte = 1; /* Number of bytes of URI args at *zUri */ ++ int nUri = 0; /* Number of URI parameters */ + + /* Figure out how much space is required for each journal file-handle + ** (there are two of them, the main journal and the sub-journal). */ +@@ -55372,14 +56311,24 @@ + } + zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ + rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); ++ if( rc!=SQLITE_OK ){ ++ if( rc==SQLITE_OK_SYMLINK ){ ++ if( vfsFlags & SQLITE_OPEN_NOFOLLOW ){ ++ rc = SQLITE_CANTOPEN_SYMLINK; ++ }else{ ++ rc = SQLITE_OK; ++ } ++ } ++ } + nPathname = sqlite3Strlen30(zPathname); + z = zUri = &zFilename[sqlite3Strlen30(zFilename)+1]; + while( *z ){ +- z += sqlite3Strlen30(z)+1; +- z += sqlite3Strlen30(z)+1; ++ z += strlen(z)+1; ++ z += strlen(z)+1; ++ nUri++; + } +- nUri = (int)(&z[1] - zUri); +- assert( nUri>=0 ); ++ nUriByte = (int)(&z[1] - zUri); ++ assert( nUriByte>=1 ); + if( rc==SQLITE_OK && nPathname+8>pVfs->mxPathname ){ + /* This branch is taken when the journal path required by + ** the database being opened will be more than pVfs->mxPathname +@@ -55404,50 +56353,111 @@ + ** Database file handle (pVfs->szOsFile bytes) + ** Sub-journal file handle (journalFileSize bytes) + ** Main journal file handle (journalFileSize bytes) ++ ** Ptr back to the Pager (sizeof(Pager*) bytes) ++ ** \0\0\0\0 database prefix (4 bytes) + ** Database file name (nPathname+1 bytes) +- ** Journal file name (nPathname+8+1 bytes) ++ ** URI query parameters (nUriByte bytes) ++ ** Journal filename (nPathname+8+1 bytes) ++ ** WAL filename (nPathname+4+1 bytes) ++ ** \0\0\0 terminator (3 bytes) ++ ** ++ ** Some 3rd-party software, over which we have no control, depends on ++ ** the specific order of the filenames and the \0 separators between them ++ ** so that it can (for example) find the database filename given the WAL ++ ** filename without using the sqlite3_filename_database() API. This is a ++ ** misuse of SQLite and a bug in the 3rd-party software, but the 3rd-party ++ ** software is in widespread use, so we try to avoid changing the filename ++ ** order and formatting if possible. In particular, the details of the ++ ** filename format expected by 3rd-party software should be as follows: ++ ** ++ ** - Main Database Path ++ ** - \0 ++ ** - Multiple URI components consisting of: ++ ** - Key ++ ** - \0 ++ ** - Value ++ ** - \0 ++ ** - \0 ++ ** - Journal Path ++ ** - \0 ++ ** - WAL Path (zWALName) ++ ** - \0 ++ ** ++ ** The sqlite3_create_filename() interface and the databaseFilename() utility ++ ** that is used by sqlite3_filename_database() and kin also depend on the ++ ** specific formatting and order of the various filenames, so if the format ++ ** changes here, be sure to change it there as well. + */ + pPtr = (u8 *)sqlite3MallocZero( +- ROUND8(sizeof(*pPager)) + /* Pager structure */ +- ROUND8(pcacheSize) + /* PCache object */ +- ROUND8(pVfs->szOsFile) + /* The main db file */ +- journalFileSize * 2 + /* The two journal files */ +- nPathname + 1 + nUri + /* zFilename */ +- nPathname + 8 + 2 /* zJournal */ ++ ROUND8(sizeof(*pPager)) + /* Pager structure */ ++ ROUND8(pcacheSize) + /* PCache object */ ++ ROUND8(pVfs->szOsFile) + /* The main db file */ ++ journalFileSize * 2 + /* The two journal files */ ++ sizeof(pPager) + /* Space to hold a pointer */ ++ 4 + /* Database prefix */ ++ nPathname + 1 + /* database filename */ ++ nUriByte + /* query parameters */ ++ nPathname + 8 + 1 + /* Journal filename */ + #ifndef SQLITE_OMIT_WAL +- + nPathname + 4 + 2 /* zWal */ ++ nPathname + 4 + 1 + /* WAL filename */ + #endif ++ 3 /* Terminator */ + ); + assert( EIGHT_BYTE_ALIGNMENT(SQLITE_INT_TO_PTR(journalFileSize)) ); + if( !pPtr ){ + sqlite3DbFree(0, zPathname); + return SQLITE_NOMEM_BKPT; + } +- pPager = (Pager*)(pPtr); +- pPager->pPCache = (PCache*)(pPtr += ROUND8(sizeof(*pPager))); +- pPager->fd = (sqlite3_file*)(pPtr += ROUND8(pcacheSize)); +- pPager->sjfd = (sqlite3_file*)(pPtr += ROUND8(pVfs->szOsFile)); +- pPager->jfd = (sqlite3_file*)(pPtr += journalFileSize); +- pPager->zFilename = (char*)(pPtr += journalFileSize); ++ pPager = (Pager*)pPtr; pPtr += ROUND8(sizeof(*pPager)); ++ pPager->pPCache = (PCache*)pPtr; pPtr += ROUND8(pcacheSize); ++ pPager->fd = (sqlite3_file*)pPtr; pPtr += ROUND8(pVfs->szOsFile); ++ pPager->sjfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; ++ pPager->jfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; + assert( EIGHT_BYTE_ALIGNMENT(pPager->jfd) ); ++ memcpy(pPtr, &pPager, sizeof(pPager)); pPtr += sizeof(pPager); ++ ++ /* Fill in the Pager.zFilename and pPager.zQueryParam fields */ ++ pPtr += 4; /* Skip zero prefix */ ++ pPager->zFilename = (char*)pPtr; ++ if( nPathname>0 ){ ++ memcpy(pPtr, zPathname, nPathname); pPtr += nPathname + 1; ++ if( zUri ){ ++ memcpy(pPtr, zUri, nUriByte); pPtr += nUriByte; ++ }else{ ++ pPtr++; ++ } ++ } ++ ++ ++ /* Fill in Pager.zJournal */ ++ if( nPathname>0 ){ ++ pPager->zJournal = (char*)pPtr; ++ memcpy(pPtr, zPathname, nPathname); pPtr += nPathname; ++ memcpy(pPtr, "-journal",8); pPtr += 8 + 1; ++#ifdef SQLITE_ENABLE_8_3_NAMES ++ sqlite3FileSuffix3(zFilename,pPager->zJournal); ++ pPtr = (u8*)(pPager->zJournal + sqlite3Strlen30(pPager->zJournal)+1); ++#endif ++ }else{ ++ pPager->zJournal = 0; ++ } + +- /* Fill in the Pager.zFilename and Pager.zJournal buffers, if required. */ +- if( zPathname ){ +- assert( nPathname>0 ); +- pPager->zJournal = (char*)(pPtr += nPathname + 1 + nUri); +- memcpy(pPager->zFilename, zPathname, nPathname); +- if( nUri ) memcpy(&pPager->zFilename[nPathname+1], zUri, nUri); +- memcpy(pPager->zJournal, zPathname, nPathname); +- memcpy(&pPager->zJournal[nPathname], "-journal\000", 8+2); +- sqlite3FileSuffix3(pPager->zFilename, pPager->zJournal); + #ifndef SQLITE_OMIT_WAL +- pPager->zWal = &pPager->zJournal[nPathname+8+1]; +- memcpy(pPager->zWal, zPathname, nPathname); +- memcpy(&pPager->zWal[nPathname], "-wal\000", 4+1); +- sqlite3FileSuffix3(pPager->zFilename, pPager->zWal); ++ /* Fill in Pager.zWal */ ++ if( nPathname>0 ){ ++ pPager->zWal = (char*)pPtr; ++ memcpy(pPtr, zPathname, nPathname); pPtr += nPathname; ++ memcpy(pPtr, "-wal", 4); pPtr += 4 + 1; ++#ifdef SQLITE_ENABLE_8_3_NAMES ++ sqlite3FileSuffix3(zFilename, pPager->zWal); ++ pPtr = (u8*)(pPager->zWal + sqlite3Strlen30(pPager->zWal)+1); + #endif +- sqlite3DbFree(0, zPathname); ++ }else{ ++ pPager->zWal = 0; + } ++#endif ++ ++ if( nPathname ) sqlite3DbFree(0, zPathname); + pPager->pVfs = pVfs; + pPager->vfsFlags = vfsFlags; + +@@ -55496,9 +56506,9 @@ + } + #endif + } +- pPager->noLock = sqlite3_uri_boolean(zFilename, "nolock", 0); ++ pPager->noLock = sqlite3_uri_boolean(pPager->zFilename, "nolock", 0); + if( (iDc & SQLITE_IOCAP_IMMUTABLE)!=0 +- || sqlite3_uri_boolean(zFilename, "immutable", 0) ){ ++ || sqlite3_uri_boolean(pPager->zFilename, "immutable", 0) ){ + vfsFlags |= SQLITE_OPEN_READONLY; + goto act_like_temp_file; + } +@@ -55605,6 +56615,19 @@ + return SQLITE_OK; + } + ++/* ++** Return the sqlite3_file for the main database given the name ++** of the corresonding WAL or Journal name as passed into ++** xOpen. ++*/ ++SQLITE_API sqlite3_file *sqlite3_database_file_object(const char *zName){ ++ Pager *pPager; ++ while( zName[-1]!=0 || zName[-2]!=0 || zName[-3]!=0 || zName[-4]!=0 ){ ++ zName--; ++ } ++ pPager = *(Pager**)(zName - 4 - sizeof(Pager*)); ++ return pPager->fd; ++} + + + /* +@@ -56160,9 +57183,6 @@ + ); + + assert( USEFETCH(pPager) ); +-#ifdef SQLITE_HAS_CODEC +- assert( pPager->xCodec==0 ); +-#endif + + /* Optimization note: Adding the "pgno<=1" term before "pgno==0" here + ** allows the compiler optimizer to reuse the results of the "pgno>1" +@@ -56293,7 +57313,6 @@ + assert( pPg->pgno==1 ); + assert( (pPg->flags & PGHDR_MMAP)==0 ); /* Page1 is never memory mapped */ + pPager = pPg->pPager; +- sqlite3PagerResetLockTimeout(pPager); + sqlite3PcacheRelease(pPg); + pagerUnlockIfUnused(pPager); + } +@@ -56491,7 +57510,7 @@ + assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); + + assert( pPager->journalHdr<=pPager->journalOff ); +- CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM_BKPT, pData2); ++ pData2 = pPg->pData; + cksum = pager_cksum(pPager, (u8*)pData2); + + /* Even if an IO or diskfull error occurs while journalling the +@@ -56856,7 +57875,7 @@ + if( DIRECT_MODE ){ + const void *zBuf; + assert( pPager->dbFileSize>0 ); +- CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM_BKPT, zBuf); ++ zBuf = pPgHdr->pData; + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); + pPager->aStat[PAGER_STAT_WRITE]++; +@@ -57189,6 +58208,7 @@ + ** But if (due to a coding error elsewhere in the system) it does get + ** called, just return the same error code without doing anything. */ + if( NEVER(pPager->errCode) ) return pPager->errCode; ++ pPager->iDataVersion++; + + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_FINISHED +@@ -57217,7 +58237,6 @@ + } + + PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); +- pPager->iDataVersion++; + rc = pager_end_transaction(pPager, pPager->setMaster, 1); + return pager_error(pPager, rc); + } +@@ -57561,9 +58580,13 @@ + ** behavior. But when the Btree needs to know the filename for matching to + ** shared cache, it uses nullIfMemDb==0 so that in-memory databases can + ** participate in shared-cache. ++** ++** The return value to this routine is always safe to use with ++** sqlite3_uri_parameter() and sqlite3_filename_database() and friends. + */ +-SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager *pPager, int nullIfMemDb){ +- return (nullIfMemDb && pPager->memDb) ? "" : pPager->zFilename; ++SQLITE_PRIVATE const char *sqlite3PagerFilename(const Pager *pPager, int nullIfMemDb){ ++ static const char zFake[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; ++ return (nullIfMemDb && pPager->memDb) ? &zFake[4] : pPager->zFilename; + } + + /* +@@ -57582,16 +58605,6 @@ + return pPager->fd; + } + +-#ifdef SQLITE_ENABLE_SETLK_TIMEOUT +-/* +-** Reset the lock timeout for pager. +-*/ +-SQLITE_PRIVATE void sqlite3PagerResetLockTimeout(Pager *pPager){ +- int x = 0; +- sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_LOCK_TIMEOUT, &x); +-} +-#endif +- + /* + ** Return the file handle for the journal file (if it exists). + ** This will be either the rollback journal or the WAL file. +@@ -57611,54 +58624,6 @@ + return pPager->zJournal; + } + +-#ifdef SQLITE_HAS_CODEC +-/* +-** Set or retrieve the codec for this pager +-*/ +-SQLITE_PRIVATE void sqlite3PagerSetCodec( +- Pager *pPager, +- void *(*xCodec)(void*,void*,Pgno,int), +- void (*xCodecSizeChng)(void*,int,int), +- void (*xCodecFree)(void*), +- void *pCodec +-){ +- if( pPager->xCodecFree ){ +- pPager->xCodecFree(pPager->pCodec); +- }else{ +- pager_reset(pPager); +- } +- pPager->xCodec = pPager->memDb ? 0 : xCodec; +- pPager->xCodecSizeChng = xCodecSizeChng; +- pPager->xCodecFree = xCodecFree; +- pPager->pCodec = pCodec; +- setGetterMethod(pPager); +- pagerReportSize(pPager); +-} +-SQLITE_PRIVATE void *sqlite3PagerGetCodec(Pager *pPager){ +- return pPager->pCodec; +-} +- +-/* +-** This function is called by the wal module when writing page content +-** into the log file. +-** +-** This function returns a pointer to a buffer containing the encrypted +-** page content. If a malloc fails, this function may return NULL. +-*/ +-SQLITE_PRIVATE void *sqlite3PagerCodec(PgHdr *pPg){ +- void *aData = 0; +- CODEC2(pPg->pPager, pPg->pData, pPg->pgno, 6, return 0, aData); +- return aData; +-} +- +-/* +-** Return the current pager state +-*/ +-SQLITE_PRIVATE int sqlite3PagerState(Pager *pPager){ +- return pPager->eState; +-} +-#endif /* SQLITE_HAS_CODEC */ +- + #ifndef SQLITE_OMIT_AUTOVACUUM + /* + ** Move the page pPg to location pgno in the file. +@@ -58053,7 +59018,6 @@ + pPager->walSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, + pnLog, pnCkpt + ); +- sqlite3PagerResetLockTimeout(pPager); + } + return rc; + } +@@ -58218,6 +59182,32 @@ + return rc; + } + ++#ifdef SQLITE_ENABLE_SETLK_TIMEOUT ++/* ++** If pager pPager is a wal-mode database not in exclusive locking mode, ++** invoke the sqlite3WalWriteLock() function on the associated Wal object ++** with the same db and bLock parameters as were passed to this function. ++** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise. ++*/ ++SQLITE_PRIVATE int sqlite3PagerWalWriteLock(Pager *pPager, int bLock){ ++ int rc = SQLITE_OK; ++ if( pagerUseWal(pPager) && pPager->exclusiveMode==0 ){ ++ rc = sqlite3WalWriteLock(pPager->pWal, bLock); ++ } ++ return rc; ++} ++ ++/* ++** Set the database handle used by the wal layer to determine if ++** blocking locks are required. ++*/ ++SQLITE_PRIVATE void sqlite3PagerWalDb(Pager *pPager, sqlite3 *db){ ++ if( pagerUseWal(pPager) ){ ++ sqlite3WalDb(pPager->pWal, db); ++ } ++} ++#endif ++ + #ifdef SQLITE_ENABLE_SNAPSHOT + /* + ** If this is a WAL database, obtain a snapshot handle for the snapshot +@@ -58236,7 +59226,10 @@ + ** read transaction is opened, attempt to read from the snapshot it + ** identifies. If this is not a WAL database, return an error. + */ +-SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot){ ++SQLITE_PRIVATE int sqlite3PagerSnapshotOpen( ++ Pager *pPager, ++ sqlite3_snapshot *pSnapshot ++){ + int rc = SQLITE_OK; + if( pPager->pWal ){ + sqlite3WalSnapshotOpen(pPager->pWal, pSnapshot); +@@ -58572,18 +59565,6 @@ + # define WALTRACE(X) + #endif + +-/* +-** WAL mode depends on atomic aligned 32-bit loads and stores in a few +-** places. The following macros try to make this explicit. +-*/ +-#if GCC_VESRION>=5004000 +-# define AtomicLoad(PTR) __atomic_load_n((PTR),__ATOMIC_RELAXED) +-# define AtomicStore(PTR,VAL) __atomic_store_n((PTR),(VAL),__ATOMIC_RELAXED) +-#else +-# define AtomicLoad(PTR) (*(PTR)) +-# define AtomicStore(PTR,VAL) (*(PTR) = (VAL)) +-#endif +- + /* + ** The maximum (and only) versions of the wal and wal-index formats + ** that may be interpreted by this version of SQLite. +@@ -58793,6 +59774,9 @@ + #ifdef SQLITE_ENABLE_SNAPSHOT + WalIndexHdr *pSnapshot; /* Start transaction here if not NULL */ + #endif ++#ifdef SQLITE_ENABLE_SETLK_TIMEOUT ++ sqlite3 *db; ++#endif + }; + + /* +@@ -58891,7 +59875,7 @@ + if( pWal->nWiData<=iPage ){ + sqlite3_int64 nByte = sizeof(u32*)*(iPage+1); + volatile u32 **apNew; +- apNew = (volatile u32 **)sqlite3_realloc64((void *)pWal->apWiData, nByte); ++ apNew = (volatile u32 **)sqlite3Realloc((void *)pWal->apWiData, nByte); + if( !apNew ){ + *ppPage = 0; + return SQLITE_NOMEM_BKPT; +@@ -59012,18 +59996,35 @@ + aOut[1] = s2; + } + ++/* ++** If there is the possibility of concurrent access to the SHM file ++** from multiple threads and/or processes, then do a memory barrier. ++*/ + static void walShmBarrier(Wal *pWal){ + if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){ + sqlite3OsShmBarrier(pWal->pDbFd); + } + } + ++/* ++** Add the SQLITE_NO_TSAN as part of the return-type of a function ++** definition as a hint that the function contains constructs that ++** might give false-positive TSAN warnings. ++** ++** See tag-20200519-1. ++*/ ++#if defined(__clang__) && !defined(SQLITE_NO_TSAN) ++# define SQLITE_NO_TSAN __attribute__((no_sanitize_thread)) ++#else ++# define SQLITE_NO_TSAN ++#endif ++ + /* + ** Write the header information in pWal->hdr into the wal-index. + ** + ** The checksum on pWal->hdr is updated before it is written. + */ +-static void walIndexWriteHdr(Wal *pWal){ ++static SQLITE_NO_TSAN void walIndexWriteHdr(Wal *pWal){ + volatile WalIndexHdr *aHdr = walIndexHdr(pWal); + const int nCksum = offsetof(WalIndexHdr, aCksum); + +@@ -59031,6 +60032,7 @@ + pWal->hdr.isInit = 1; + pWal->hdr.iVersion = WALINDEX_MAX_VERSION; + walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum); ++ /* Possible TSAN false-positive. See tag-20200519-1 */ + memcpy((void*)&aHdr[1], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); + walShmBarrier(pWal); + memcpy((void*)&aHdr[0], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); +@@ -59166,7 +60168,7 @@ + SQLITE_SHM_LOCK | SQLITE_SHM_SHARED); + WALTRACE(("WAL%p: acquire SHARED-%s %s\n", pWal, + walLockName(lockIdx), rc ? "failed" : "ok")); +- VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && rc!=SQLITE_BUSY); ) ++ VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && (rc&0xFF)!=SQLITE_BUSY); ) + return rc; + } + static void walUnlockShared(Wal *pWal, int lockIdx){ +@@ -59182,7 +60184,7 @@ + SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE); + WALTRACE(("WAL%p: acquire EXCLUSIVE-%s cnt=%d %s\n", pWal, + walLockName(lockIdx), n, rc ? "failed" : "ok")); +- VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && rc!=SQLITE_BUSY); ) ++ VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && (rc&0xFF)!=SQLITE_BUSY); ) + return rc; + } + static void walUnlockExclusive(Wal *pWal, int lockIdx, int n){ +@@ -60002,6 +61004,89 @@ + return rc; + } + ++#ifdef SQLITE_ENABLE_SETLK_TIMEOUT ++/* ++** Attempt to enable blocking locks. Blocking locks are enabled only if (a) ++** they are supported by the VFS, and (b) the database handle is configured ++** with a busy-timeout. Return 1 if blocking locks are successfully enabled, ++** or 0 otherwise. ++*/ ++static int walEnableBlocking(Wal *pWal){ ++ int res = 0; ++ if( pWal->db ){ ++ int tmout = pWal->db->busyTimeout; ++ if( tmout ){ ++ int rc; ++ rc = sqlite3OsFileControl( ++ pWal->pDbFd, SQLITE_FCNTL_LOCK_TIMEOUT, (void*)&tmout ++ ); ++ res = (rc==SQLITE_OK); ++ } ++ } ++ return res; ++} ++ ++/* ++** Disable blocking locks. ++*/ ++static void walDisableBlocking(Wal *pWal){ ++ int tmout = 0; ++ sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_LOCK_TIMEOUT, (void*)&tmout); ++} ++ ++/* ++** If parameter bLock is true, attempt to enable blocking locks, take ++** the WRITER lock, and then disable blocking locks. If blocking locks ++** cannot be enabled, no attempt to obtain the WRITER lock is made. Return ++** an SQLite error code if an error occurs, or SQLITE_OK otherwise. It is not ++** an error if blocking locks can not be enabled. ++** ++** If the bLock parameter is false and the WRITER lock is held, release it. ++*/ ++SQLITE_PRIVATE int sqlite3WalWriteLock(Wal *pWal, int bLock){ ++ int rc = SQLITE_OK; ++ assert( pWal->readLock<0 || bLock==0 ); ++ if( bLock ){ ++ assert( pWal->db ); ++ if( walEnableBlocking(pWal) ){ ++ rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); ++ if( rc==SQLITE_OK ){ ++ pWal->writeLock = 1; ++ } ++ walDisableBlocking(pWal); ++ } ++ }else if( pWal->writeLock ){ ++ walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); ++ pWal->writeLock = 0; ++ } ++ return rc; ++} ++ ++/* ++** Set the database handle used to determine if blocking locks are required. ++*/ ++SQLITE_PRIVATE void sqlite3WalDb(Wal *pWal, sqlite3 *db){ ++ pWal->db = db; ++} ++ ++/* ++** Take an exclusive WRITE lock. Blocking if so configured. ++*/ ++static int walLockWriter(Wal *pWal){ ++ int rc; ++ walEnableBlocking(pWal); ++ rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); ++ walDisableBlocking(pWal); ++ return rc; ++} ++#else ++# define walEnableBlocking(x) 0 ++# define walDisableBlocking(x) ++# define walLockWriter(pWal) walLockExclusive((pWal), WAL_WRITE_LOCK, 1) ++# define sqlite3WalDb(pWal, db) ++#endif /* ifdef SQLITE_ENABLE_SETLK_TIMEOUT */ ++ ++ + /* + ** Attempt to obtain the exclusive WAL lock defined by parameters lockIdx and + ** n. If the attempt fails and parameter xBusy is not NULL, then it is a +@@ -60019,6 +61104,12 @@ + do { + rc = walLockExclusive(pWal, lockIdx, n); + }while( xBusy && rc==SQLITE_BUSY && xBusy(pBusyArg) ); ++#ifdef SQLITE_ENABLE_SETLK_TIMEOUT ++ if( rc==SQLITE_BUSY_TIMEOUT ){ ++ walDisableBlocking(pWal); ++ rc = SQLITE_BUSY; ++ } ++#endif + return rc; + } + +@@ -60056,7 +61147,7 @@ + sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); + memcpy(&pWal->hdr.aSalt[1], &salt1, 4); + walIndexWriteHdr(pWal); +- pInfo->nBackfill = 0; ++ AtomicStore(&pInfo->nBackfill, 0); + pInfo->nBackfillAttempted = 0; + pInfo->aReadMark[1] = 0; + for(i=2; i<WAL_NREADER; i++) pInfo->aReadMark[i] = READMARK_NOT_USED; +@@ -60131,20 +61222,13 @@ + mxSafeFrame = pWal->hdr.mxFrame; + mxPage = pWal->hdr.nPage; + for(i=1; i<WAL_NREADER; i++){ +- /* Thread-sanitizer reports that the following is an unsafe read, +- ** as some other thread may be in the process of updating the value +- ** of the aReadMark[] slot. The assumption here is that if that is +- ** happening, the other client may only be increasing the value, +- ** not decreasing it. So assuming either that either the "old" or +- ** "new" version of the value is read, and not some arbitrary value +- ** that would never be written by a real client, things are still +- ** safe. */ +- u32 y = pInfo->aReadMark[i]; ++ u32 y = AtomicLoad(pInfo->aReadMark+i); + if( mxSafeFrame>y ){ + assert( y<=pWal->hdr.mxFrame ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); + if( rc==SQLITE_OK ){ +- pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED); ++ u32 iMark = (i==1 ? mxSafeFrame : READMARK_NOT_USED); ++ AtomicStore(pInfo->aReadMark+i, iMark); + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + }else if( rc==SQLITE_BUSY ){ + mxSafeFrame = y; +@@ -60162,7 +61246,7 @@ + } + + if( pIter +- && (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0),1))==SQLITE_OK ++ && (rc = walBusyLock(pWal,xBusy,pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK + ){ + u32 nBackfill = pInfo->nBackfill; + +@@ -60177,6 +61261,7 @@ + if( rc==SQLITE_OK ){ + i64 nReq = ((i64)mxPage * szPage); + i64 nSize; /* Current size of database file */ ++ sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_START, 0); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSize<nReq ){ + sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); +@@ -60188,7 +61273,7 @@ + while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ + i64 iOffset; + assert( walFramePgno(pWal, iFrame)==iDbpage ); +- if( db->u1.isInterrupted ){ ++ if( AtomicLoad(&db->u1.isInterrupted) ){ + rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT; + break; + } +@@ -60204,6 +61289,7 @@ + rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); + if( rc!=SQLITE_OK ) break; + } ++ sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_DONE, 0); + + /* If work was actually accomplished... */ + if( rc==SQLITE_OK ){ +@@ -60216,7 +61302,7 @@ + } + } + if( rc==SQLITE_OK ){ +- pInfo->nBackfill = mxSafeFrame; ++ AtomicStore(&pInfo->nBackfill, mxSafeFrame); + } + } + +@@ -60375,7 +61461,7 @@ + ** If the checksum cannot be verified return non-zero. If the header + ** is read successfully and the checksum verified, return zero. + */ +-static int walIndexTryHdr(Wal *pWal, int *pChanged){ ++static SQLITE_NO_TSAN int walIndexTryHdr(Wal *pWal, int *pChanged){ + u32 aCksum[2]; /* Checksum on the header content */ + WalIndexHdr h1, h2; /* Two copies of the header content */ + WalIndexHdr volatile *aHdr; /* Header in shared memory */ +@@ -60388,13 +61474,19 @@ + ** meaning it is possible that an inconsistent snapshot is read + ** from the file. If this happens, return non-zero. + ** ++ ** tag-20200519-1: + ** There are two copies of the header at the beginning of the wal-index. + ** When reading, read [0] first then [1]. Writes are in the reverse order. + ** Memory barriers are used to prevent the compiler or the hardware from +- ** reordering the reads and writes. ++ ** reordering the reads and writes. TSAN and similar tools can sometimes ++ ** give false-positive warnings about these accesses because the tools do not ++ ** account for the double-read and the memory barrier. The use of mutexes ++ ** here would be problematic as the memory being accessed is potentially ++ ** shared among multiple processes and not all mutex implementions work ++ ** reliably in that environment. + */ + aHdr = walIndexHdr(pWal); +- memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); ++ memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); /* Possible TSAN false-positive */ + walShmBarrier(pWal); + memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); + +@@ -60484,28 +61576,32 @@ + /* If the first attempt failed, it might have been due to a race + ** with a writer. So get a WRITE lock and try again. + */ +- assert( badHdr==0 || pWal->writeLock==0 ); + if( badHdr ){ + if( pWal->bShmUnreliable==0 && (pWal->readOnly & WAL_SHM_RDONLY) ){ + if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){ + walUnlockShared(pWal, WAL_WRITE_LOCK); + rc = SQLITE_READONLY_RECOVERY; + } +- }else if( SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ +- pWal->writeLock = 1; +- if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ +- badHdr = walIndexTryHdr(pWal, pChanged); +- if( badHdr ){ +- /* If the wal-index header is still malformed even while holding +- ** a WRITE lock, it can only mean that the header is corrupted and +- ** needs to be reconstructed. So run recovery to do exactly that. +- */ +- rc = walIndexRecover(pWal); +- *pChanged = 1; ++ }else{ ++ int bWriteLock = pWal->writeLock; ++ if( bWriteLock || SQLITE_OK==(rc = walLockWriter(pWal)) ){ ++ pWal->writeLock = 1; ++ if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ ++ badHdr = walIndexTryHdr(pWal, pChanged); ++ if( badHdr ){ ++ /* If the wal-index header is still malformed even while holding ++ ** a WRITE lock, it can only mean that the header is corrupted and ++ ** needs to be reconstructed. So run recovery to do exactly that. ++ */ ++ rc = walIndexRecover(pWal); ++ *pChanged = 1; ++ } ++ } ++ if( bWriteLock==0 ){ ++ pWal->writeLock = 0; ++ walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + } + } +- pWal->writeLock = 0; +- walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + } + } + +@@ -60835,7 +61931,7 @@ + assert( pWal->nWiData>0 ); + assert( pWal->apWiData[0]!=0 ); + pInfo = walCkptInfo(pWal); +- if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame ++ if( !useWal && AtomicLoad(&pInfo->nBackfill)==pWal->hdr.mxFrame + #ifdef SQLITE_ENABLE_SNAPSHOT + && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0) + #endif +@@ -60897,7 +61993,8 @@ + for(i=1; i<WAL_NREADER; i++){ + rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1); + if( rc==SQLITE_OK ){ +- mxReadMark = AtomicStore(pInfo->aReadMark+i,mxFrame); ++ AtomicStore(pInfo->aReadMark+i,mxFrame); ++ mxReadMark = mxFrame; + mxI = i; + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + break; +@@ -61001,7 +62098,7 @@ + rc = SQLITE_NOMEM; + }else{ + u32 i = pInfo->nBackfillAttempted; +- for(i=pInfo->nBackfillAttempted; i>pInfo->nBackfill; i--){ ++ for(i=pInfo->nBackfillAttempted; i>AtomicLoad(&pInfo->nBackfill); i--){ + WalHashLoc sLoc; /* Hash table location */ + u32 pgno; /* Page number in db file */ + i64 iDbOff; /* Offset of db file entry */ +@@ -61056,12 +62153,35 @@ + SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ + int rc; /* Return code */ + int cnt = 0; /* Number of TryBeginRead attempts */ +- + #ifdef SQLITE_ENABLE_SNAPSHOT + int bChanged = 0; + WalIndexHdr *pSnapshot = pWal->pSnapshot; +- if( pSnapshot && memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){ +- bChanged = 1; ++#endif ++ ++ assert( pWal->ckptLock==0 ); ++ ++#ifdef SQLITE_ENABLE_SNAPSHOT ++ if( pSnapshot ){ ++ if( memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){ ++ bChanged = 1; ++ } ++ ++ /* It is possible that there is a checkpointer thread running ++ ** concurrent with this code. If this is the case, it may be that the ++ ** checkpointer has already determined that it will checkpoint ++ ** snapshot X, where X is later in the wal file than pSnapshot, but ++ ** has not yet set the pInfo->nBackfillAttempted variable to indicate ++ ** its intent. To avoid the race condition this leads to, ensure that ++ ** there is no checkpointer process by taking a shared CKPT lock ++ ** before checking pInfo->nBackfillAttempted. */ ++ (void)walEnableBlocking(pWal); ++ rc = walLockShared(pWal, WAL_CKPT_LOCK); ++ walDisableBlocking(pWal); ++ ++ if( rc!=SQLITE_OK ){ ++ return rc; ++ } ++ pWal->ckptLock = 1; + } + #endif + +@@ -61094,48 +62214,42 @@ + assert( pWal->readLock>0 || pWal->hdr.mxFrame==0 ); + assert( pInfo->aReadMark[pWal->readLock]<=pSnapshot->mxFrame ); + +- /* It is possible that there is a checkpointer thread running +- ** concurrent with this code. If this is the case, it may be that the +- ** checkpointer has already determined that it will checkpoint +- ** snapshot X, where X is later in the wal file than pSnapshot, but +- ** has not yet set the pInfo->nBackfillAttempted variable to indicate +- ** its intent. To avoid the race condition this leads to, ensure that +- ** there is no checkpointer process by taking a shared CKPT lock +- ** before checking pInfo->nBackfillAttempted. +- ** +- ** TODO: Does the aReadMark[] lock prevent a checkpointer from doing +- ** this already? +- */ +- rc = walLockShared(pWal, WAL_CKPT_LOCK); +- +- if( rc==SQLITE_OK ){ +- /* Check that the wal file has not been wrapped. Assuming that it has +- ** not, also check that no checkpointer has attempted to checkpoint any +- ** frames beyond pSnapshot->mxFrame. If either of these conditions are +- ** true, return SQLITE_ERROR_SNAPSHOT. Otherwise, overwrite pWal->hdr +- ** with *pSnapshot and set *pChanged as appropriate for opening the +- ** snapshot. */ +- if( !memcmp(pSnapshot->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) +- && pSnapshot->mxFrame>=pInfo->nBackfillAttempted +- ){ +- assert( pWal->readLock>0 ); +- memcpy(&pWal->hdr, pSnapshot, sizeof(WalIndexHdr)); +- *pChanged = bChanged; +- }else{ +- rc = SQLITE_ERROR_SNAPSHOT; +- } +- +- /* Release the shared CKPT lock obtained above. */ +- walUnlockShared(pWal, WAL_CKPT_LOCK); +- pWal->minFrame = 1; ++ /* Check that the wal file has not been wrapped. Assuming that it has ++ ** not, also check that no checkpointer has attempted to checkpoint any ++ ** frames beyond pSnapshot->mxFrame. If either of these conditions are ++ ** true, return SQLITE_ERROR_SNAPSHOT. Otherwise, overwrite pWal->hdr ++ ** with *pSnapshot and set *pChanged as appropriate for opening the ++ ** snapshot. */ ++ if( !memcmp(pSnapshot->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) ++ && pSnapshot->mxFrame>=pInfo->nBackfillAttempted ++ ){ ++ assert( pWal->readLock>0 ); ++ memcpy(&pWal->hdr, pSnapshot, sizeof(WalIndexHdr)); ++ *pChanged = bChanged; ++ }else{ ++ rc = SQLITE_ERROR_SNAPSHOT; + } + ++ /* A client using a non-current snapshot may not ignore any frames ++ ** from the start of the wal file. This is because, for a system ++ ** where (minFrame < iSnapshot < maxFrame), a checkpointer may ++ ** have omitted to checkpoint a frame earlier than minFrame in ++ ** the file because there exists a frame after iSnapshot that ++ ** is the same database page. */ ++ pWal->minFrame = 1; + + if( rc!=SQLITE_OK ){ + sqlite3WalEndReadTransaction(pWal); + } + } + } ++ ++ /* Release the shared CKPT lock obtained above. */ ++ if( pWal->ckptLock ){ ++ assert( pSnapshot ); ++ walUnlockShared(pWal, WAL_CKPT_LOCK); ++ pWal->ckptLock = 0; ++ } + #endif + return rc; + } +@@ -61215,22 +62329,24 @@ + int iKey; /* Hash slot index */ + int nCollide; /* Number of hash collisions remaining */ + int rc; /* Error code */ ++ u32 iH; + + rc = walHashGet(pWal, iHash, &sLoc); + if( rc!=SQLITE_OK ){ + return rc; + } + nCollide = HASHTABLE_NSLOT; +- for(iKey=walHash(pgno); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){ +- u32 iFrame = sLoc.aHash[iKey] + sLoc.iZero; +- if( iFrame<=iLast && iFrame>=pWal->minFrame +- && sLoc.aPgno[sLoc.aHash[iKey]]==pgno ){ ++ iKey = walHash(pgno); ++ while( (iH = AtomicLoad(&sLoc.aHash[iKey]))!=0 ){ ++ u32 iFrame = iH + sLoc.iZero; ++ if( iFrame<=iLast && iFrame>=pWal->minFrame && sLoc.aPgno[iH]==pgno ){ + assert( iFrame>iRead || CORRUPT_DB ); + iRead = iFrame; + } + if( (nCollide--)==0 ){ + return SQLITE_CORRUPT_BKPT; + } ++ iKey = walNextHash(iKey); + } + if( iRead ) break; + } +@@ -61306,6 +62422,16 @@ + SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal){ + int rc; + ++#ifdef SQLITE_ENABLE_SETLK_TIMEOUT ++ /* If the write-lock is already held, then it was obtained before the ++ ** read-transaction was even opened, making this call a no-op. ++ ** Return early. */ ++ if( pWal->writeLock ){ ++ assert( !memcmp(&pWal->hdr,(void *)walIndexHdr(pWal),sizeof(WalIndexHdr)) ); ++ return SQLITE_OK; ++ } ++#endif ++ + /* Cannot start a write transaction without first holding a read + ** transaction. */ + assert( pWal->readLock>=0 ); +@@ -61551,11 +62677,7 @@ + int rc; /* Result code from subfunctions */ + void *pData; /* Data actually written */ + u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ +-#if defined(SQLITE_HAS_CODEC) +- if( (pData = sqlite3PagerCodec(pPage))==0 ) return SQLITE_NOMEM_BKPT; +-#else + pData = pPage->pData; +-#endif + walEncodeFrame(p->pWal, pPage->pgno, nTruncate, pData, aFrame); + rc = walWriteToLog(p, aFrame, sizeof(aFrame), iOffset); + if( rc ) return rc; +@@ -61738,11 +62860,7 @@ + if( pWal->iReCksum==0 || iWrite<pWal->iReCksum ){ + pWal->iReCksum = iWrite; + } +-#if defined(SQLITE_HAS_CODEC) +- if( (pData = sqlite3PagerCodec(p))==0 ) return SQLITE_NOMEM; +-#else + pData = p->pData; +-#endif + rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOff); + if( rc ) return rc; + p->flags &= ~PGHDR_WAL_APPEND; +@@ -61792,6 +62910,7 @@ + if( rc ) return rc; + iOffset += szFrame; + nExtra++; ++ assert( pLast!=0 ); + } + } + if( bSync ){ +@@ -61824,6 +62943,7 @@ + iFrame++; + rc = walIndexAppend(pWal, iFrame, p->pgno); + } ++ assert( pLast!=0 || nExtra==0 ); + while( rc==SQLITE_OK && nExtra>0 ){ + iFrame++; + nExtra--; +@@ -61888,45 +63008,52 @@ + if( pWal->readOnly ) return SQLITE_READONLY; + WALTRACE(("WAL%p: checkpoint begins\n", pWal)); + ++ /* Enable blocking locks, if possible. If blocking locks are successfully ++ ** enabled, set xBusy2=0 so that the busy-handler is never invoked. */ ++ sqlite3WalDb(pWal, db); ++ (void)walEnableBlocking(pWal); ++ + /* IMPLEMENTATION-OF: R-62028-47212 All calls obtain an exclusive +- ** "checkpoint" lock on the database file. */ ++ ** "checkpoint" lock on the database file. ++ ** EVIDENCE-OF: R-10421-19736 If any other process is running a ++ ** checkpoint operation at the same time, the lock cannot be obtained and ++ ** SQLITE_BUSY is returned. ++ ** EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, ++ ** it will not be invoked in this case. ++ */ + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); +- if( rc ){ +- /* EVIDENCE-OF: R-10421-19736 If any other process is running a +- ** checkpoint operation at the same time, the lock cannot be obtained and +- ** SQLITE_BUSY is returned. +- ** EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, +- ** it will not be invoked in this case. +- */ +- testcase( rc==SQLITE_BUSY ); +- testcase( xBusy!=0 ); +- return rc; +- } +- pWal->ckptLock = 1; ++ testcase( rc==SQLITE_BUSY ); ++ testcase( rc!=SQLITE_OK && xBusy2!=0 ); ++ if( rc==SQLITE_OK ){ ++ pWal->ckptLock = 1; + +- /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and +- ** TRUNCATE modes also obtain the exclusive "writer" lock on the database +- ** file. +- ** +- ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained +- ** immediately, and a busy-handler is configured, it is invoked and the +- ** writer lock retried until either the busy-handler returns 0 or the +- ** lock is successfully obtained. +- */ +- if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ +- rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_WRITE_LOCK, 1); +- if( rc==SQLITE_OK ){ +- pWal->writeLock = 1; +- }else if( rc==SQLITE_BUSY ){ +- eMode2 = SQLITE_CHECKPOINT_PASSIVE; +- xBusy2 = 0; +- rc = SQLITE_OK; ++ /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and ++ ** TRUNCATE modes also obtain the exclusive "writer" lock on the database ++ ** file. ++ ** ++ ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained ++ ** immediately, and a busy-handler is configured, it is invoked and the ++ ** writer lock retried until either the busy-handler returns 0 or the ++ ** lock is successfully obtained. ++ */ ++ if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ ++ rc = walBusyLock(pWal, xBusy2, pBusyArg, WAL_WRITE_LOCK, 1); ++ if( rc==SQLITE_OK ){ ++ pWal->writeLock = 1; ++ }else if( rc==SQLITE_BUSY ){ ++ eMode2 = SQLITE_CHECKPOINT_PASSIVE; ++ xBusy2 = 0; ++ rc = SQLITE_OK; ++ } + } + } + ++ + /* Read the wal-index header. */ + if( rc==SQLITE_OK ){ ++ walDisableBlocking(pWal); + rc = walIndexReadHdr(pWal, &isChanged); ++ (void)walEnableBlocking(pWal); + if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ + sqlite3OsUnfetch(pWal->pDbFd, 0, 0); + } +@@ -61958,11 +63085,19 @@ + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); + } + ++ walDisableBlocking(pWal); ++ sqlite3WalDb(pWal, 0); ++ + /* Release the locks. */ + sqlite3WalEndWriteTransaction(pWal); +- walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); +- pWal->ckptLock = 0; ++ if( pWal->ckptLock ){ ++ walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); ++ pWal->ckptLock = 0; ++ } + WALTRACE(("WAL%p: checkpoint %s\n", pWal, rc ? "failed" : "ok")); ++#ifdef SQLITE_ENABLE_SETLK_TIMEOUT ++ if( rc==SQLITE_BUSY_TIMEOUT ) rc = SQLITE_BUSY; ++#endif + return (rc==SQLITE_OK && eMode!=eMode2 ? SQLITE_BUSY : rc); + } + +@@ -62079,7 +63214,10 @@ + + /* Try to open on pSnapshot when the next read-transaction starts + */ +-SQLITE_PRIVATE void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot){ ++SQLITE_PRIVATE void sqlite3WalSnapshotOpen( ++ Wal *pWal, ++ sqlite3_snapshot *pSnapshot ++){ + pWal->pSnapshot = (WalIndexHdr*)pSnapshot; + } + +@@ -62598,9 +63736,7 @@ + #endif + u8 inTransaction; /* Transaction state */ + u8 max1bytePayload; /* Maximum first byte of cell for a 1-byte payload */ +-#ifdef SQLITE_HAS_CODEC +- u8 optimalReserve; /* Desired amount of reserved space per page */ +-#endif ++ u8 nReserveWanted; /* Desired number of extra bytes per page */ + u16 btsFlags; /* Boolean parameters. See BTS_* macros below */ + u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */ + u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */ +@@ -62723,6 +63859,7 @@ + #define BTCF_AtLast 0x08 /* Cursor is pointing ot the last entry */ + #define BTCF_Incrblob 0x10 /* True if an incremental I/O handle */ + #define BTCF_Multiple 0x20 /* Maybe another cursor on the same btree */ ++#define BTCF_Pinned 0x40 /* Cursor is busy and cannot be moved */ + + /* + ** Potential values for BtCursor.eState. +@@ -62866,6 +64003,7 @@ + int v1, v2; /* Values for up to two %d fields in zPfx */ + StrAccum errMsg; /* Accumulate the error message text here */ + u32 *heap; /* Min-heap used for analyzing cell coverage */ ++ sqlite3 *db; /* Database connection running the check */ + }; + + /* +@@ -63799,7 +64937,7 @@ + */ + static int btreeGetHasContent(BtShared *pBt, Pgno pgno){ + Bitvec *p = pBt->pHasContent; +- return (p && (pgno>sqlite3BitvecSize(p) || sqlite3BitvecTest(p, pgno))); ++ return p && (pgno>sqlite3BitvecSize(p) || sqlite3BitvecTestNotNull(p, pgno)); + } + + /* +@@ -63887,6 +65025,9 @@ + assert( 0==pCur->pKey ); + assert( cursorHoldsMutex(pCur) ); + ++ if( pCur->curFlags & BTCF_Pinned ){ ++ return SQLITE_CONSTRAINT_PINNED; ++ } + if( pCur->eState==CURSOR_SKIPNEXT ){ + pCur->eState = CURSOR_VALID; + }else{ +@@ -64643,7 +65784,7 @@ + if( iFree2+sz2 > usableSize ) return SQLITE_CORRUPT_PAGE(pPage); + memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz)); + sz += sz2; +- }else if( iFree+sz>usableSize ){ ++ }else if( NEVER(iFree+sz>usableSize) ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + +@@ -64816,7 +65957,7 @@ + ** However, that integer is too large to be stored in a 2-byte unsigned + ** integer, so a value of 0 is used in its place. */ + top = get2byte(&data[hdr+5]); +- assert( top<=(int)pPage->pBt->usableSize ); /* Prevent by getAndInitPage() */ ++ assert( top<=(int)pPage->pBt->usableSize ); /* by btreeComputeFreeSpace() */ + if( gap>top ){ + if( top==0 && pPage->pBt->usableSize==65536 ){ + top = 65536; +@@ -64835,9 +65976,14 @@ + if( (data[hdr+2] || data[hdr+1]) && gap+2<=top ){ + u8 *pSpace = pageFindSlot(pPage, nByte, &rc); + if( pSpace ){ +- assert( pSpace>=data && (pSpace - data)<65536 ); +- *pIdx = (int)(pSpace - data); +- return SQLITE_OK; ++ int g2; ++ assert( pSpace+nByte<=data+pPage->pBt->usableSize ); ++ *pIdx = g2 = (int)(pSpace-data); ++ if( NEVER(g2<=gap) ){ ++ return SQLITE_CORRUPT_PAGE(pPage); ++ }else{ ++ return SQLITE_OK; ++ } + }else if( rc ){ + return rc; + } +@@ -64911,12 +66057,12 @@ + }else{ + while( (iFreeBlk = get2byte(&data[iPtr]))<iStart ){ + if( iFreeBlk<iPtr+4 ){ +- if( iFreeBlk==0 ) break; ++ if( iFreeBlk==0 ) break; /* TH3: corrupt082.100 */ + return SQLITE_CORRUPT_PAGE(pPage); + } + iPtr = iFreeBlk; + } +- if( iFreeBlk>pPage->pBt->usableSize-4 ){ ++ if( iFreeBlk>pPage->pBt->usableSize-4 ){ /* TH3: corrupt081.100 */ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( iFreeBlk>iPtr || iFreeBlk==0 ); +@@ -64931,7 +66077,7 @@ + nFrag = iFreeBlk - iEnd; + if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_PAGE(pPage); + iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); +- if( iEnd > pPage->pBt->usableSize ){ ++ if( NEVER(iEnd > pPage->pBt->usableSize) ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + iSize = iEnd - iStart; +@@ -64959,7 +66105,8 @@ + /* The new freeblock is at the beginning of the cell content area, + ** so just extend the cell content area rather than create another + ** freelist entry */ +- if( iStart<x || iPtr!=hdr+1 ) return SQLITE_CORRUPT_PAGE(pPage); ++ if( iStart<x ) return SQLITE_CORRUPT_PAGE(pPage); ++ if( iPtr!=hdr+1 ) return SQLITE_CORRUPT_PAGE(pPage); + put2byte(&data[hdr+1], iFreeBlk); + put2byte(&data[hdr+5], iEnd); + }else{ +@@ -65079,7 +66226,7 @@ + nFree = data[hdr+7] + top; /* Init nFree to non-freeblock free space */ + if( pc>0 ){ + u32 next, size; +- if( pc<iCellFirst ){ ++ if( pc<top ){ + /* EVIDENCE-OF: R-55530-52930 In a well-formed b-tree page, there will + ** always be at least one cell before the first freeblock. + */ +@@ -65113,7 +66260,7 @@ + ** serves to verify that the offset to the start of the cell-content + ** area, according to the page header, lies within the page. + */ +- if( nFree>usableSize ){ ++ if( nFree>usableSize || nFree<iCellFirst ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + pPage->nFree = (u16)(nFree - iCellFirst); +@@ -65316,12 +66463,12 @@ + ** error, return ((unsigned int)-1). + */ + static Pgno btreePagecount(BtShared *pBt){ ++ assert( (pBt->nPage & 0x80000000)==0 || CORRUPT_DB ); + return pBt->nPage; + } + SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree *p){ + assert( sqlite3BtreeHoldsMutex(p) ); +- assert( ((p->pBt->nPage)&0x80000000)==0 ); +- return btreePagecount(p->pBt); ++ return btreePagecount(p->pBt) & 0x7fffffff; + } + + /* +@@ -65483,8 +66630,7 @@ + BtShared *pBt = (BtShared*)pArg; + assert( pBt->db ); + assert( sqlite3_mutex_held(pBt->db->mutex) ); +- return sqlite3InvokeBusyHandler(&pBt->db->busyHandler, +- sqlite3PagerFile(pBt->pPager)); ++ return sqlite3InvokeBusyHandler(&pBt->db->busyHandler); + } + + /* +@@ -65588,9 +66734,13 @@ + rc = sqlite3OsFullPathname(pVfs, zFilename, + nFullPathname, zFullPathname); + if( rc ){ +- sqlite3_free(zFullPathname); +- sqlite3_free(p); +- return rc; ++ if( rc==SQLITE_OK_SYMLINK ){ ++ rc = SQLITE_OK; ++ }else{ ++ sqlite3_free(zFullPathname); ++ sqlite3_free(p); ++ return rc; ++ } + } + } + #if SQLITE_THREADSAFE +@@ -66031,19 +67181,17 @@ + */ + SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, int iFix){ + int rc = SQLITE_OK; ++ int x; + BtShared *pBt = p->pBt; +- assert( nReserve>=-1 && nReserve<=255 ); ++ assert( nReserve>=0 && nReserve<=255 ); + sqlite3BtreeEnter(p); +-#if SQLITE_HAS_CODEC +- if( nReserve>pBt->optimalReserve ) pBt->optimalReserve = (u8)nReserve; +-#endif ++ pBt->nReserveWanted = nReserve; ++ x = pBt->pageSize - pBt->usableSize; ++ if( nReserve<x ) nReserve = x; + if( pBt->btsFlags & BTS_PAGESIZE_FIXED ){ + sqlite3BtreeLeave(p); + return SQLITE_READONLY; + } +- if( nReserve<0 ){ +- nReserve = pBt->pageSize - pBt->usableSize; +- } + assert( nReserve>=0 && nReserve<=255 ); + if( pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE && + ((pageSize-1)&pageSize)==0 ){ +@@ -66089,19 +67237,17 @@ + ** are intentually left unused. This is the "reserved" space that is + ** sometimes used by extensions. + ** +-** If SQLITE_HAS_MUTEX is defined then the number returned is the +-** greater of the current reserved space and the maximum requested +-** reserve space. ++** The value returned is the larger of the current reserve size and ++** the latest reserve size requested by SQLITE_FILECTRL_RESERVE_BYTES. ++** The amount of reserve can only grow - never shrink. + */ +-SQLITE_PRIVATE int sqlite3BtreeGetOptimalReserve(Btree *p){ +- int n; ++SQLITE_PRIVATE int sqlite3BtreeGetRequestedReserve(Btree *p){ ++ int n1, n2; + sqlite3BtreeEnter(p); +- n = sqlite3BtreeGetReserveNoMutex(p); +-#ifdef SQLITE_HAS_CODEC +- if( n<p->pBt->optimalReserve ) n = p->pBt->optimalReserve; +-#endif ++ n1 = (int)p->pBt->nReserveWanted; ++ n2 = sqlite3BtreeGetReserveNoMutex(p); + sqlite3BtreeLeave(p); +- return n; ++ return n1>n2 ? n1 : n2; + } + + +@@ -66551,6 +67697,7 @@ + */ + SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ + BtShared *pBt = p->pBt; ++ Pager *pPager = pBt->pPager; + int rc = SQLITE_OK; + + sqlite3BtreeEnter(p); +@@ -66566,7 +67713,7 @@ + assert( pBt->inTransaction==TRANS_WRITE || IfNotOmitAV(pBt->bDoTruncate)==0 ); + + if( (p->db->flags & SQLITE_ResetDatabase) +- && sqlite3PagerIsreadonly(pBt->pPager)==0 ++ && sqlite3PagerIsreadonly(pPager)==0 + ){ + pBt->btsFlags &= ~BTS_READ_ONLY; + } +@@ -66614,6 +67761,18 @@ + pBt->btsFlags &= ~BTS_INITIALLY_EMPTY; + if( pBt->nPage==0 ) pBt->btsFlags |= BTS_INITIALLY_EMPTY; + do { ++ sqlite3PagerWalDb(pPager, p->db); ++ ++#ifdef SQLITE_ENABLE_SETLK_TIMEOUT ++ /* If transitioning from no transaction directly to a write transaction, ++ ** block for the WRITER lock first if possible. */ ++ if( pBt->pPage1==0 && wrflag ){ ++ assert( pBt->inTransaction==TRANS_NONE ); ++ rc = sqlite3PagerWalWriteLock(pPager, 1); ++ if( rc!=SQLITE_BUSY && rc!=SQLITE_OK ) break; ++ } ++#endif ++ + /* Call lockBtree() until either pBt->pPage1 is populated or + ** lockBtree() returns something other than SQLITE_OK. lockBtree() + ** may return SQLITE_OK but leave pBt->pPage1 set to 0 if after +@@ -66627,7 +67786,7 @@ + if( (pBt->btsFlags & BTS_READ_ONLY)!=0 ){ + rc = SQLITE_READONLY; + }else{ +- rc = sqlite3PagerBegin(pBt->pPager,wrflag>1,sqlite3TempInMemory(p->db)); ++ rc = sqlite3PagerBegin(pPager, wrflag>1, sqlite3TempInMemory(p->db)); + if( rc==SQLITE_OK ){ + rc = newDatabase(pBt); + }else if( rc==SQLITE_BUSY_SNAPSHOT && pBt->inTransaction==TRANS_NONE ){ +@@ -66640,11 +67799,15 @@ + } + + if( rc!=SQLITE_OK ){ ++ (void)sqlite3PagerWalWriteLock(pPager, 0); + unlockBtreeIfUnused(pBt); + } + }while( (rc&0xFF)==SQLITE_BUSY && pBt->inTransaction==TRANS_NONE && + btreeInvokeBusyHandler(pBt) ); +- sqlite3PagerResetLockTimeout(pBt->pPager); ++ sqlite3PagerWalDb(pPager, 0); ++#ifdef SQLITE_ENABLE_SETLK_TIMEOUT ++ if( rc==SQLITE_BUSY_TIMEOUT ) rc = SQLITE_BUSY; ++#endif + + if( rc==SQLITE_OK ){ + if( p->inTrans==TRANS_NONE ){ +@@ -66696,7 +67859,7 @@ + ** open savepoints. If the second parameter is greater than 0 and + ** the sub-journal is not already open, then it will be opened here. + */ +- rc = sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint); ++ rc = sqlite3PagerOpenSavepoint(pPager, p->db->nSavepoint); + } + } + +@@ -67341,6 +68504,18 @@ + return rc; + } + ++/* ++** Set the pBt->nPage field correctly, according to the current ++** state of the database. Assume pBt->pPage1 is valid. ++*/ ++static void btreeSetNPage(BtShared *pBt, MemPage *pPage1){ ++ int nPage = get4byte(&pPage1->aData[28]); ++ testcase( nPage==0 ); ++ if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); ++ testcase( pBt->nPage!=nPage ); ++ pBt->nPage = nPage; ++} ++ + /* + ** Rollback the transaction in progress. + ** +@@ -67386,11 +68561,7 @@ + ** call btreeGetPage() on page 1 again to make + ** sure pPage1->aData is set correctly. */ + if( btreeGetPage(pBt, 1, &pPage1, 0)==SQLITE_OK ){ +- int nPage = get4byte(28+(u8*)pPage1->aData); +- testcase( nPage==0 ); +- if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); +- testcase( pBt->nPage!=nPage ); +- pBt->nPage = nPage; ++ btreeSetNPage(pBt, pPage1); + releasePageOne(pPage1); + } + assert( countValidCursors(pBt, 1)==0 ); +@@ -67470,12 +68641,11 @@ + pBt->nPage = 0; + } + rc = newDatabase(pBt); +- pBt->nPage = get4byte(28 + pBt->pPage1->aData); ++ btreeSetNPage(pBt, pBt->pPage1); + +- /* The database size was written into the offset 28 of the header +- ** when the transaction started, so we know that the value at offset +- ** 28 is nonzero. */ +- assert( pBt->nPage>0 ); ++ /* pBt->nPage might be zero if the database was corrupt when ++ ** the transaction was started. Otherwise, it must be at least 1. */ ++ assert( CORRUPT_DB || pBt->nPage>0 ); + } + sqlite3BtreeLeave(p); + } +@@ -67543,8 +68713,9 @@ + /* The following assert statements verify that if this is a sharable + ** b-tree database, the connection is holding the required table locks, + ** and that no other connection has any open cursor that conflicts with +- ** this lock. */ +- assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, (wrFlag?2:1)) ); ++ ** this lock. The iTable<1 term disables the check for corrupt schemas. */ ++ assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, (wrFlag?2:1)) ++ || iTable<1 ); + assert( wrFlag==0 || !hasReadConflicts(p, iTable) ); + + /* Assert that the caller has opened the required transaction. */ +@@ -67557,9 +68728,13 @@ + allocateTempSpace(pBt); + if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM_BKPT; + } +- if( iTable==1 && btreePagecount(pBt)==0 ){ +- assert( wrFlag==0 ); +- iTable = 0; ++ if( iTable<=1 ){ ++ if( iTable<1 ){ ++ return SQLITE_CORRUPT_BKPT; ++ }else if( btreePagecount(pBt)==0 ){ ++ assert( wrFlag==0 ); ++ iTable = 0; ++ } + } + + /* Now that no other errors can occur, finish filling in the BtCursor +@@ -67584,6 +68759,19 @@ + pCur->eState = CURSOR_INVALID; + return SQLITE_OK; + } ++static int btreeCursorWithLock( ++ Btree *p, /* The btree */ ++ int iTable, /* Root page of table to open */ ++ int wrFlag, /* 1 to write. 0 read-only */ ++ struct KeyInfo *pKeyInfo, /* First arg to comparison function */ ++ BtCursor *pCur /* Space for new cursor */ ++){ ++ int rc; ++ sqlite3BtreeEnter(p); ++ rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); ++ sqlite3BtreeLeave(p); ++ return rc; ++} + SQLITE_PRIVATE int sqlite3BtreeCursor( + Btree *p, /* The btree */ + int iTable, /* Root page of table to open */ +@@ -67591,15 +68779,11 @@ + struct KeyInfo *pKeyInfo, /* First arg to xCompare() */ + BtCursor *pCur /* Write new cursor here */ + ){ +- int rc; +- if( iTable<1 ){ +- rc = SQLITE_CORRUPT_BKPT; ++ if( p->sharable ){ ++ return btreeCursorWithLock(p, iTable, wrFlag, pKeyInfo, pCur); + }else{ +- sqlite3BtreeEnter(p); +- rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); +- sqlite3BtreeLeave(p); ++ return btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); + } +- return rc; + } + + /* +@@ -67722,6 +68906,18 @@ + return pCur->info.nKey; + } + ++/* ++** Pin or unpin a cursor. ++*/ ++SQLITE_PRIVATE void sqlite3BtreeCursorPin(BtCursor *pCur){ ++ assert( (pCur->curFlags & BTCF_Pinned)==0 ); ++ pCur->curFlags |= BTCF_Pinned; ++} ++SQLITE_PRIVATE void sqlite3BtreeCursorUnpin(BtCursor *pCur){ ++ assert( (pCur->curFlags & BTCF_Pinned)!=0 ); ++ pCur->curFlags &= ~BTCF_Pinned; ++} ++ + #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC + /* + ** Return the offset into the database file for the start of the +@@ -68057,6 +69253,7 @@ + assert( aWrite>=pBufStart ); /* due to (6) */ + memcpy(aSave, aWrite, 4); + rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1)); ++ if( rc && nextPage>pBt->nPage ) rc = SQLITE_CORRUPT_BKPT; + nextPage = get4byte(aWrite); + memcpy(aWrite, aSave, 4); + }else +@@ -68483,6 +69680,7 @@ + assert( pCur->ix==pCur->pPage->nCell-1 ); + assert( pCur->pPage->leaf ); + #endif ++ *pRes = 0; + return SQLITE_OK; + } + +@@ -68704,6 +69902,7 @@ + ** case this happens. */ + void *pCellKey; + u8 * const pCellBody = pCell - pPage->childPtrSize; ++ const int nOverrun = 18; /* Size of the overrun padding */ + pPage->xParseCell(pPage, pCellBody, &pCur->info); + nCell = (int)pCur->info.nKey; + testcase( nCell<0 ); /* True if key size is 2^32 or more */ +@@ -68714,13 +69913,14 @@ + rc = SQLITE_CORRUPT_PAGE(pPage); + goto moveto_finish; + } +- pCellKey = sqlite3Malloc( nCell+18 ); ++ pCellKey = sqlite3Malloc( nCell+nOverrun ); + if( pCellKey==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto moveto_finish; + } + pCur->ix = (u16)idx; + rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 0); ++ memset(((u8*)pCellKey)+nCell,0,nOverrun); /* Fix uninit warnings */ + pCur->curFlags &= ~BTCF_ValidOvfl; + if( rc ){ + sqlite3_free(pCellKey); +@@ -68874,8 +70074,11 @@ + ** to be invalid here. This can only occur if a second cursor modifies + ** the page while cursor pCur is holding a reference to it. Which can + ** only happen if the database is corrupt in such a way as to link the +- ** page into more than one b-tree structure. */ +- testcase( idx>pPage->nCell ); ++ ** page into more than one b-tree structure. ++ ** ++ ** Update 2019-12-23: appears to long longer be possible after the ++ ** addition of anotherValidCursor() condition on balance_deeper(). */ ++ harmless( idx>pPage->nCell ); + + if( idx>=pPage->nCell ){ + if( !pPage->leaf ){ +@@ -69842,12 +71045,7 @@ + assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) ); + assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); +- /* The cell should normally be sized correctly. However, when moving a +- ** malformed cell from a leaf page to an interior page, if the cell size +- ** wanted to be less than 4 but got rounded up to 4 on the leaf, then size +- ** might be less than 8 (leaf-size + pointer) on the interior node. Hence +- ** the term after the || in the following assert(). */ +- assert( sz==pPage->xCellSize(pPage, pCell) || (sz==8 && iChild>0) ); ++ assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB ); + assert( pPage->nFree>=0 ); + if( pPage->nOverflow || sz+2>pPage->nFree ){ + if( pTemp ){ +@@ -70105,7 +71303,7 @@ + if( pData < pCellptr ) return SQLITE_CORRUPT_BKPT; + memcpy(pData, pCell, sz); + assert( sz==pPg->xCellSize(pPg, pCell) || CORRUPT_DB ); +- testcase( sz!=pPg->xCellSize(pPg,pCell) ); ++ testcase( sz!=pPg->xCellSize(pPg,pCell) ) + i++; + if( i>=iEnd ) break; + if( pCArray->ixNx[k]<=i ){ +@@ -70171,7 +71369,8 @@ + while( 1 /*Exit by break*/ ){ + int sz, rc; + u8 *pSlot; +- sz = cachedCellSize(pCArray, i); ++ assert( pCArray->szCell[i]!=0 ); ++ sz = pCArray->szCell[i]; + if( (aData[1]==0 && aData[2]==0) || (pSlot = pageFindSlot(pPg,sz,&rc))==0 ){ + if( (pData - pBegin)<sz ) return 1; + pData -= sz; +@@ -70296,7 +71495,7 @@ + assert( nCell>=0 ); + if( iOld<iNew ){ + int nShift = pageFreeArray(pPg, iOld, iNew-iOld, pCArray); +- if( nShift>nCell ) return SQLITE_CORRUPT_BKPT; ++ if( NEVER(nShift>nCell) ) return SQLITE_CORRUPT_BKPT; + memmove(pPg->aCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); + nCell -= nShift; + } +@@ -70332,6 +71531,7 @@ + memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + } + nCell++; ++ cachedCellSize(pCArray, iCell+iNew); + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + iCell+iNew, 1, pCArray +@@ -70825,6 +72025,7 @@ + u16 maskPage = pOld->maskPage; + u8 *piCell = aData + pOld->cellOffset; + u8 *piEnd; ++ VVA_ONLY( int nCellAtStart = b.nCell; ) + + /* Verify that all sibling pages are of the same "type" (table-leaf, + ** table-interior, index-leaf, or index-interior). +@@ -70853,6 +72054,10 @@ + */ + memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow)); + if( pOld->nOverflow>0 ){ ++ if( NEVER(limit<pOld->aiOvfl[0]) ){ ++ rc = SQLITE_CORRUPT_BKPT; ++ goto balance_cleanup; ++ } + limit = pOld->aiOvfl[0]; + for(j=0; j<limit; j++){ + b.apCell[b.nCell] = aData + (maskPage & get2byteAligned(piCell)); +@@ -70872,6 +72077,7 @@ + piCell += 2; + b.nCell++; + } ++ assert( (b.nCell-nCellAtStart)==(pOld->nCell+pOld->nOverflow) ); + + cntOld[i] = b.nCell; + if( i<nOld-1 && !leafData){ +@@ -71134,6 +72340,8 @@ + )); + + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); ++ assert( nNew>=1 && nNew<=ArraySize(apNew) ); ++ assert( apNew[nNew-1]!=0 ); + put4byte(pRight, apNew[nNew-1]->pgno); + + /* If the sibling pages are not leaves, ensure that the right-child pointer +@@ -71172,6 +72380,7 @@ + while( i==cntOldNext ){ + iOld++; + assert( iOld<nNew || iOld<nOld ); ++ assert( iOld>=0 && iOld<NB ); + pOld = iOld<nNew ? apNew[iOld] : apOld[iOld]; + cntOldNext += pOld->nCell + pOld->nOverflow + !leafData; + } +@@ -71458,6 +72667,30 @@ + return SQLITE_OK; + } + ++/* ++** Return SQLITE_CORRUPT if any cursor other than pCur is currently valid ++** on the same B-tree as pCur. ++** ++** This can if a database is corrupt with two or more SQL tables ++** pointing to the same b-tree. If an insert occurs on one SQL table ++** and causes a BEFORE TRIGGER to do a secondary insert on the other SQL ++** table linked to the same b-tree. If the secondary insert causes a ++** rebalance, that can change content out from under the cursor on the ++** first SQL table, violating invariants on the first insert. ++*/ ++static int anotherValidCursor(BtCursor *pCur){ ++ BtCursor *pOther; ++ for(pOther=pCur->pBt->pCursor; pOther; pOther=pOther->pNext){ ++ if( pOther!=pCur ++ && pOther->eState==CURSOR_VALID ++ && pOther->pPage==pCur->pPage ++ ){ ++ return SQLITE_CORRUPT_BKPT; ++ } ++ } ++ return SQLITE_OK; ++} ++ + /* + ** The page that pCur currently points to has just been modified in + ** some way. This function figures out if this modification means the +@@ -71478,12 +72711,14 @@ + VVA_ONLY( int balance_deeper_called = 0 ); + + do { +- int iPage = pCur->iPage; ++ int iPage; + MemPage *pPage = pCur->pPage; + + if( NEVER(pPage->nFree<0) && btreeComputeFreeSpace(pPage) ) break; +- if( iPage==0 ){ +- if( pPage->nOverflow ){ ++ if( pPage->nOverflow==0 && pPage->nFree<=nMin ){ ++ break; ++ }else if( (iPage = pCur->iPage)==0 ){ ++ if( pPage->nOverflow && (rc = anotherValidCursor(pCur))==SQLITE_OK ){ + /* The root page of the b-tree is overfull. In this case call the + ** balance_deeper() function to create a new child for the root-page + ** and copy the current contents of the root-page to it. The +@@ -71503,8 +72738,6 @@ + }else{ + break; + } +- }else if( pPage->nOverflow==0 && pPage->nFree<=nMin ){ +- break; + }else{ + MemPage * const pParent = pCur->apPage[iPage-1]; + int const iIdx = pCur->aiIdx[iPage-1]; +@@ -71646,7 +72879,9 @@ + Pgno ovflPgno; /* Next overflow page to write */ + u32 ovflPageSize; /* Size to write on overflow page */ + +- if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd ){ ++ if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd ++ || pCur->info.pPayload < pPage->aData + pPage->cellOffset ++ ){ + return SQLITE_CORRUPT_BKPT; + } + /* Overwrite the local portion first */ +@@ -71779,7 +73014,6 @@ + if( flags & BTREE_SAVEPOSITION ){ + assert( pCur->curFlags & BTCF_ValidNKey ); + assert( pX->nKey==pCur->info.nKey ); +- assert( pCur->info.nSize!=0 ); + assert( loc==0 ); + } + #endif +@@ -71854,7 +73088,9 @@ + } + + } +- assert( pCur->eState==CURSOR_VALID || (pCur->eState==CURSOR_INVALID && loc) ); ++ assert( pCur->eState==CURSOR_VALID ++ || (pCur->eState==CURSOR_INVALID && loc) ++ || CORRUPT_DB ); + + pPage = pCur->pPage; + assert( pPage->intKey || pX->nKey>=0 ); +@@ -71887,6 +73123,8 @@ + memcpy(newCell, oldCell, 4); + } + rc = clearCell(pPage, oldCell, &info); ++ testcase( pCur->curFlags & BTCF_ValidOvfl ); ++ invalidateOverflowCache(pCur); + if( info.nSize==szNew && info.nLocal==info.nPayload + && (!ISAUTOVACUUM || szNew<pPage->minLocal) + ){ +@@ -71900,7 +73138,12 @@ + ** new entry uses overflow pages, as the insertCell() call below is + ** necessary to add the PTRMAP_OVERFLOW1 pointer-map entry. */ + assert( rc==SQLITE_OK ); /* clearCell never fails when nLocal==nPayload */ +- if( oldCell+szNew > pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT; ++ if( oldCell < pPage->aData+pPage->hdrOffset+10 ){ ++ return SQLITE_CORRUPT_BKPT; ++ } ++ if( oldCell+szNew > pPage->aDataEnd ){ ++ return SQLITE_CORRUPT_BKPT; ++ } + memcpy(oldCell, newCell, szNew); + return SQLITE_OK; + } +@@ -72609,7 +73852,6 @@ + return rc; + } + +-#ifndef SQLITE_OMIT_BTREECOUNT + /* + ** The first argument, pCur, is a cursor opened on some b-tree. Count the + ** number of entries in the b-tree and write the result to *pnEntry. +@@ -72618,7 +73860,7 @@ + ** Otherwise, if an error is encountered (i.e. an IO error or database + ** corruption) an SQLite error code is returned. + */ +-SQLITE_PRIVATE int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){ ++SQLITE_PRIVATE int sqlite3BtreeCount(sqlite3 *db, BtCursor *pCur, i64 *pnEntry){ + i64 nEntry = 0; /* Value to return in *pnEntry */ + int rc; /* Return code */ + +@@ -72631,7 +73873,7 @@ + /* Unless an error occurs, the following loop runs one iteration for each + ** page in the B-Tree structure (not including overflow pages). + */ +- while( rc==SQLITE_OK ){ ++ while( rc==SQLITE_OK && !AtomicLoad(&db->u1.isInterrupted) ){ + int iIdx; /* Index of child node in parent */ + MemPage *pPage; /* Current page of the b-tree */ + +@@ -72682,7 +73924,6 @@ + /* An error has occurred. Return an error code. */ + return rc; + } +-#endif + + /* + ** Return the pager associated with a BTree. This routine is used for +@@ -72757,6 +73998,7 @@ + checkAppendMsg(pCheck, "2nd reference to page %d", iPage); + return 1; + } ++ if( AtomicLoad(&pCheck->db->u1.isInterrupted) ) return 1; + setPageReferenced(pCheck, iPage); + return 0; + } +@@ -73200,6 +74442,7 @@ + ** returned. If a memory allocation error occurs, NULL is returned. + */ + SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( ++ sqlite3 *db, /* Database connection that is running the check */ + Btree *p, /* The btree to be checked */ + int *aRoot, /* An array of root pages numbers for individual trees */ + int nRoot, /* Number of entries in aRoot[] */ +@@ -73217,6 +74460,7 @@ + assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE ); + VVA_ONLY( nRef = sqlite3PagerRefcount(pBt->pPager) ); + assert( nRef>=0 ); ++ sCheck.db = db; + sCheck.pBt = pBt; + sCheck.pPager = pBt->pPager; + sCheck.nPage = btreePagecount(sCheck.pBt); +@@ -73730,7 +74974,7 @@ + */ + static int setDestPgsz(sqlite3_backup *p){ + int rc; +- rc = sqlite3BtreeSetPageSize(p->pDest,sqlite3BtreeGetPageSize(p->pSrc),-1,0); ++ rc = sqlite3BtreeSetPageSize(p->pDest,sqlite3BtreeGetPageSize(p->pSrc),0,0); + return rc; + } + +@@ -73853,13 +75097,6 @@ + int nDestPgsz = sqlite3BtreeGetPageSize(p->pDest); + const int nCopy = MIN(nSrcPgsz, nDestPgsz); + const i64 iEnd = (i64)iSrcPg*(i64)nSrcPgsz; +-#ifdef SQLITE_HAS_CODEC +- /* Use BtreeGetReserveNoMutex() for the source b-tree, as although it is +- ** guaranteed that the shared-mutex is held by this thread, handle +- ** p->pSrc may not actually be the owner. */ +- int nSrcReserve = sqlite3BtreeGetReserveNoMutex(p->pSrc); +- int nDestReserve = sqlite3BtreeGetOptimalReserve(p->pDest); +-#endif + int rc = SQLITE_OK; + i64 iOff; + +@@ -73876,26 +75113,6 @@ + rc = SQLITE_READONLY; + } + +-#ifdef SQLITE_HAS_CODEC +- /* Backup is not possible if the page size of the destination is changing +- ** and a codec is in use. +- */ +- if( nSrcPgsz!=nDestPgsz && sqlite3PagerGetCodec(pDestPager)!=0 ){ +- rc = SQLITE_READONLY; +- } +- +- /* Backup is not possible if the number of bytes of reserve space differ +- ** between source and destination. If there is a difference, try to +- ** fix the destination to agree with the source. If that is not possible, +- ** then the backup cannot proceed. +- */ +- if( nSrcReserve!=nDestReserve ){ +- u32 newPgsz = nSrcPgsz; +- rc = sqlite3PagerSetPagesize(pDestPager, &newPgsz, nSrcReserve); +- if( rc==SQLITE_OK && newPgsz!=nSrcPgsz ) rc = SQLITE_READONLY; +- } +-#endif +- + /* This loop runs once for each destination page spanned by the source + ** page. For each iteration, variable iOff is set to the byte offset + ** of the destination page. +@@ -74237,8 +75454,10 @@ + } + if( p->isAttached ){ + pp = sqlite3PagerBackupPtr(sqlite3BtreePager(p->pSrc)); ++ assert( pp!=0 ); + while( *pp!=p ){ + pp = &(*pp)->pNext; ++ assert( pp!=0 ); + } + *pp = p->pNext; + } +@@ -74389,10 +75608,6 @@ + b.pDest = pTo; + b.iNext = 1; + +-#ifdef SQLITE_HAS_CODEC +- sqlite3PagerAlignReserve(sqlite3BtreePager(pTo), sqlite3BtreePager(pFrom)); +-#endif +- + /* 0x7FFFFFFF is the hard limit for the number of pages in a database + ** file. By passing this as the number of pages to copy to + ** sqlite3_backup_step(), we can guarantee that the copy finishes +@@ -74439,6 +75654,11 @@ + /* #include "sqliteInt.h" */ + /* #include "vdbeInt.h" */ + ++/* True if X is a power of two. 0 is considered a power of two here. ++** In other words, return true if X has at most one bit set. ++*/ ++#define ISPOWEROF2(X) (((X)&((X)-1))==0) ++ + #ifdef SQLITE_DEBUG + /* + ** Check invariants on a Mem object. +@@ -74458,8 +75678,8 @@ + ** That saves a few cycles in inner loops. */ + assert( (p->flags & MEM_Dyn)==0 || p->szMalloc==0 ); + +- /* Cannot be both MEM_Int and MEM_Real at the same time */ +- assert( (p->flags & (MEM_Int|MEM_Real))!=(MEM_Int|MEM_Real) ); ++ /* Cannot have more than one of MEM_Int, MEM_Real, or MEM_IntReal */ ++ assert( ISPOWEROF2(p->flags & (MEM_Int|MEM_Real|MEM_IntReal)) ); + + if( p->flags & MEM_Null ){ + /* Cannot be both MEM_Null and some other type */ +@@ -74513,9 +75733,31 @@ + } + #endif + ++/* ++** Render a Mem object which is one of MEM_Int, MEM_Real, or MEM_IntReal ++** into a buffer. ++*/ ++static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ ++ StrAccum acc; ++ assert( p->flags & (MEM_Int|MEM_Real|MEM_IntReal) ); ++ sqlite3StrAccumInit(&acc, 0, zBuf, sz, 0); ++ if( p->flags & MEM_Int ){ ++ sqlite3_str_appendf(&acc, "%lld", p->u.i); ++ }else if( p->flags & MEM_IntReal ){ ++ sqlite3_str_appendf(&acc, "%!.15g", (double)p->u.i); ++ }else{ ++ sqlite3_str_appendf(&acc, "%!.15g", p->u.r); ++ } ++ assert( acc.zText==zBuf && acc.mxAlloc<=0 ); ++ zBuf[acc.nChar] = 0; /* Fast version of sqlite3StrAccumFinish(&acc) */ ++} ++ + #ifdef SQLITE_DEBUG + /* +-** Check that string value of pMem agrees with its integer or real value. ++** Validity checks on pMem. pMem holds a string. ++** ++** (1) Check that string value of pMem agrees with its integer or real value. ++** (2) Check that the string is correctly zero terminated + ** + ** A single int or real value always converts to the same strings. But + ** many different strings can be converted into the same int or real. +@@ -74533,17 +75775,24 @@ + ** + ** This routine is for use inside of assert() statements only. + */ +-SQLITE_PRIVATE int sqlite3VdbeMemConsistentDualRep(Mem *p){ ++SQLITE_PRIVATE int sqlite3VdbeMemValidStrRep(Mem *p){ + char zBuf[100]; + char *z; + int i, j, incr; + if( (p->flags & MEM_Str)==0 ) return 1; +- if( (p->flags & (MEM_Int|MEM_Real))==0 ) return 1; +- if( p->flags & MEM_Int ){ +- sqlite3_snprintf(sizeof(zBuf),zBuf,"%lld",p->u.i); +- }else{ +- sqlite3_snprintf(sizeof(zBuf),zBuf,"%!.15g",p->u.r); +- } ++ if( p->flags & MEM_Term ){ ++ /* Insure that the string is properly zero-terminated. Pay particular ++ ** attention to the case where p->n is odd */ ++ if( p->szMalloc>0 && p->z==p->zMalloc ){ ++ assert( p->enc==SQLITE_UTF8 || p->szMalloc >= ((p->n+1)&~1)+2 ); ++ assert( p->enc!=SQLITE_UTF8 || p->szMalloc >= p->n+1 ); ++ } ++ assert( p->z[p->n]==0 ); ++ assert( p->enc==SQLITE_UTF8 || p->z[(p->n+1)&~1]==0 ); ++ assert( p->enc==SQLITE_UTF8 || p->z[((p->n+1)&~1)+1]==0 ); ++ } ++ if( (p->flags & (MEM_Int|MEM_Real|MEM_IntReal))==0 ) return 1; ++ vdbeMemRenderNum(sizeof(zBuf), zBuf, p); + z = p->z; + i = j = 0; + incr = 1; +@@ -74619,7 +75868,13 @@ + assert( pMem->szMalloc==0 + || pMem->szMalloc==sqlite3DbMallocSize(pMem->db, pMem->zMalloc) ); + if( pMem->szMalloc>0 && bPreserve && pMem->z==pMem->zMalloc ){ +- pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n); ++ if( pMem->db ){ ++ pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n); ++ }else{ ++ pMem->zMalloc = sqlite3Realloc(pMem->z, n); ++ if( pMem->zMalloc==0 ) sqlite3_free(pMem->z); ++ pMem->z = pMem->zMalloc; ++ } + bPreserve = 0; + }else{ + if( pMem->szMalloc>0 ) sqlite3DbFreeNN(pMem->db, pMem->zMalloc); +@@ -74655,8 +75910,8 @@ + ** + ** Any prior string or blob content in the pMem object may be discarded. + ** The pMem->xDel destructor is called, if it exists. Though MEM_Str +-** and MEM_Blob values may be discarded, MEM_Int, MEM_Real, and MEM_Null +-** values are preserved. ++** and MEM_Blob values may be discarded, MEM_Int, MEM_Real, MEM_IntReal, ++** and MEM_Null values are preserved. + ** + ** Return SQLITE_OK on success or an error code (probably SQLITE_NOMEM) + ** if unable to complete the resizing. +@@ -74669,20 +75924,26 @@ + } + assert( (pMem->flags & MEM_Dyn)==0 ); + pMem->z = pMem->zMalloc; +- pMem->flags &= (MEM_Null|MEM_Int|MEM_Real); ++ pMem->flags &= (MEM_Null|MEM_Int|MEM_Real|MEM_IntReal); + return SQLITE_OK; + } + + /* + ** It is already known that pMem contains an unterminated string. + ** Add the zero terminator. ++** ++** Three bytes of zero are added. In this way, there is guaranteed ++** to be a double-zero byte at an even byte boundary in order to ++** terminate a UTF16 string, even if the initial size of the buffer ++** is an odd number of bytes. + */ + static SQLITE_NOINLINE int vdbeMemAddTerminator(Mem *pMem){ +- if( sqlite3VdbeMemGrow(pMem, pMem->n+2, 1) ){ ++ if( sqlite3VdbeMemGrow(pMem, pMem->n+3, 1) ){ + return SQLITE_NOMEM_BKPT; + } + pMem->z[pMem->n] = 0; + pMem->z[pMem->n+1] = 0; ++ pMem->z[pMem->n+2] = 0; + pMem->flags |= MEM_Term; + return SQLITE_OK; + } +@@ -74756,12 +76017,12 @@ + } + + /* +-** Add MEM_Str to the set of representations for the given Mem. Numbers +-** are converted using sqlite3_snprintf(). Converting a BLOB to a string +-** is a no-op. ++** Add MEM_Str to the set of representations for the given Mem. This ++** routine is only called if pMem is a number of some kind, not a NULL ++** or a BLOB. + ** +-** Existing representations MEM_Int and MEM_Real are invalidated if +-** bForce is true but are retained if bForce is false. ++** Existing representations MEM_Int, MEM_Real, or MEM_IntReal are invalidated ++** if bForce is true but are retained if bForce is false. + ** + ** A MEM_Null value will never be passed to this function. This function is + ** used for converting values to text for returning to the user (i.e. via +@@ -74770,13 +76031,12 @@ + ** user and the latter is an internal programming error. + */ + SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ +- int fg = pMem->flags; + const int nByte = 32; + + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); +- assert( !(fg&MEM_Zero) ); +- assert( !(fg&(MEM_Str|MEM_Blob)) ); +- assert( fg&(MEM_Int|MEM_Real) ); ++ assert( !(pMem->flags&MEM_Zero) ); ++ assert( !(pMem->flags&(MEM_Str|MEM_Blob)) ); ++ assert( pMem->flags&(MEM_Int|MEM_Real|MEM_IntReal) ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + +@@ -74786,23 +76046,12 @@ + return SQLITE_NOMEM_BKPT; + } + +- /* For a Real or Integer, use sqlite3_snprintf() to produce the UTF-8 +- ** string representation of the value. Then, if the required encoding +- ** is UTF-16le or UTF-16be do a translation. +- ** +- ** FIX ME: It would be better if sqlite3_snprintf() could do UTF-16. +- */ +- if( fg & MEM_Int ){ +- sqlite3_snprintf(nByte, pMem->z, "%lld", pMem->u.i); +- }else{ +- assert( fg & MEM_Real ); +- sqlite3_snprintf(nByte, pMem->z, "%!.15g", pMem->u.r); +- } ++ vdbeMemRenderNum(nByte, pMem->z, pMem); + assert( pMem->z!=0 ); + pMem->n = sqlite3Strlen30NN(pMem->z); + pMem->enc = SQLITE_UTF8; + pMem->flags |= MEM_Str|MEM_Term; +- if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real); ++ if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); + sqlite3VdbeChangeEncoding(pMem, enc); + return SQLITE_OK; + } +@@ -74847,15 +76096,11 @@ + #ifndef SQLITE_OMIT_WINDOWFUNC + SQLITE_PRIVATE int sqlite3VdbeMemAggValue(Mem *pAccum, Mem *pOut, FuncDef *pFunc){ + sqlite3_context ctx; +- Mem t; + assert( pFunc!=0 ); + assert( pFunc->xValue!=0 ); + assert( (pAccum->flags & MEM_Null)!=0 || pFunc==pAccum->u.pDef ); + assert( pAccum->db==0 || sqlite3_mutex_held(pAccum->db->mutex) ); + memset(&ctx, 0, sizeof(ctx)); +- memset(&t, 0, sizeof(t)); +- t.flags = MEM_Null; +- t.db = pAccum->db; + sqlite3VdbeMemSetNull(pOut); + ctx.pOut = pOut; + ctx.pMem = pAccum; +@@ -74976,12 +76221,12 @@ + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + flags = pMem->flags; +- if( flags & MEM_Int ){ ++ if( flags & (MEM_Int|MEM_IntReal) ){ ++ testcase( flags & MEM_IntReal ); + return pMem->u.i; + }else if( flags & MEM_Real ){ + return doubleToInt64(pMem->u.r); +- }else if( flags & (MEM_Str|MEM_Blob) ){ +- assert( pMem->z || pMem->n==0 ); ++ }else if( (flags & (MEM_Str|MEM_Blob))!=0 && pMem->z!=0 ){ + return memIntValue(pMem); + }else{ + return 0; +@@ -75005,7 +76250,8 @@ + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + if( pMem->flags & MEM_Real ){ + return pMem->u.r; +- }else if( pMem->flags & MEM_Int ){ ++ }else if( pMem->flags & (MEM_Int|MEM_IntReal) ){ ++ testcase( pMem->flags & MEM_IntReal ); + return (double)pMem->u.i; + }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ + return memRealValue(pMem); +@@ -75020,7 +76266,8 @@ + ** Return the value ifNull if pMem is NULL. + */ + SQLITE_PRIVATE int sqlite3VdbeBooleanValue(Mem *pMem, int ifNull){ +- if( pMem->flags & MEM_Int ) return pMem->u.i!=0; ++ testcase( pMem->flags & MEM_IntReal ); ++ if( pMem->flags & (MEM_Int|MEM_IntReal) ) return pMem->u.i!=0; + if( pMem->flags & MEM_Null ) return ifNull; + return sqlite3VdbeRealValue(pMem)!=0.0; + } +@@ -75083,17 +76330,21 @@ + /* Compare a floating point value to an integer. Return true if the two + ** values are the same within the precision of the floating point value. + ** ++** This function assumes that i was obtained by assignment from r1. ++** + ** For some versions of GCC on 32-bit machines, if you do the more obvious + ** comparison of "r1==(double)i" you sometimes get an answer of false even + ** though the r1 and (double)i values are bit-for-bit the same. + */ +-static int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){ ++SQLITE_PRIVATE int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){ + double r2 = (double)i; +- return memcmp(&r1, &r2, sizeof(r1))==0; ++ return r1==0.0 ++ || (memcmp(&r1, &r2, sizeof(r1))==0 ++ && i >= -2251799813685248LL && i < 2251799813685248LL); + } + + /* +-** Convert pMem so that it has types MEM_Real or MEM_Int or both. ++** Convert pMem so that it has type MEM_Real or MEM_Int. + ** Invalidate any prior representations. + ** + ** Every effort is made to force the conversion, even if the input +@@ -75101,25 +76352,26 @@ + ** as much of the string as we can and ignore the rest. + */ + SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ +- if( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 ){ ++ testcase( pMem->flags & MEM_Int ); ++ testcase( pMem->flags & MEM_Real ); ++ testcase( pMem->flags & MEM_IntReal ); ++ testcase( pMem->flags & MEM_Null ); ++ if( (pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null))==0 ){ + int rc; ++ sqlite3_int64 ix; + assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); +- rc = sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc); +- if( rc==0 ){ ++ rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); ++ if( ((rc==0 || rc==1) && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1) ++ || sqlite3RealSameAsInt(pMem->u.r, (ix = (i64)pMem->u.r)) ++ ){ ++ pMem->u.i = ix; + MemSetTypeFlag(pMem, MEM_Int); + }else{ +- i64 i = pMem->u.i; +- sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); +- if( rc==1 && sqlite3RealSameAsInt(pMem->u.r, i) ){ +- pMem->u.i = i; +- MemSetTypeFlag(pMem, MEM_Int); +- }else{ +- MemSetTypeFlag(pMem, MEM_Real); +- } ++ MemSetTypeFlag(pMem, MEM_Real); + } + } +- assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))!=0 ); ++ assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null))!=0 ); + pMem->flags &= ~(MEM_Str|MEM_Blob|MEM_Zero); + return SQLITE_OK; + } +@@ -75131,8 +76383,8 @@ + ** affinity even if that results in loss of data. This routine is + ** used (for example) to implement the SQL "cast()" operator. + */ +-SQLITE_PRIVATE void sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ +- if( pMem->flags & MEM_Null ) return; ++SQLITE_PRIVATE int sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ ++ if( pMem->flags & MEM_Null ) return SQLITE_OK; + switch( aff ){ + case SQLITE_AFF_BLOB: { /* Really a cast to BLOB */ + if( (pMem->flags & MEM_Blob)==0 ){ +@@ -75162,10 +76414,11 @@ + pMem->flags |= (pMem->flags&MEM_Blob)>>3; + sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); + assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); +- pMem->flags &= ~(MEM_Int|MEM_Real|MEM_Blob|MEM_Zero); +- break; ++ pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero); ++ return sqlite3VdbeChangeEncoding(pMem, encoding); + } + } ++ return SQLITE_OK; + } + + /* +@@ -75331,25 +76584,27 @@ + ** its link to a shallow copy and by marking any current shallow + ** copies of this cell as invalid. + ** +-** This is used for testing and debugging only - to make sure shallow +-** copies are not misused. ++** This is used for testing and debugging only - to help ensure that shallow ++** copies (created by OP_SCopy) are not misused. + */ + SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ + int i; + Mem *pX; +- for(i=0, pX=pVdbe->aMem; i<pVdbe->nMem; i++, pX++){ ++ for(i=1, pX=pVdbe->aMem+1; i<pVdbe->nMem; i++, pX++){ + if( pX->pScopyFrom==pMem ){ +- /* If pX is marked as a shallow copy of pMem, then verify that ++ u16 mFlags; ++ if( pVdbe->db->flags & SQLITE_VdbeTrace ){ ++ sqlite3DebugPrintf("Invalidate R[%d] due to change in R[%d]\n", ++ (int)(pX - pVdbe->aMem), (int)(pMem - pVdbe->aMem)); ++ } ++ /* If pX is marked as a shallow copy of pMem, then try to verify that + ** no significant changes have been made to pX since the OP_SCopy. + ** A significant change would indicated a missed call to this + ** function for pX. Minor changes, such as adding or removing a + ** dual type, are allowed, as long as the underlying value is the + ** same. */ +- u16 mFlags = pMem->flags & pX->flags & pX->mScopyFlags; +- assert( (mFlags&MEM_Int)==0 || pMem->u.i==pX->u.i ); +- assert( (mFlags&MEM_Real)==0 || pMem->u.r==pX->u.r ); +- assert( (mFlags&MEM_Str)==0 || (pMem->n==pX->n && pMem->z==pX->z) ); +- assert( (mFlags&MEM_Blob)==0 || sqlite3BlobCompare(pMem,pX)==0 ); ++ mFlags = pMem->flags & pX->flags & pX->mScopyFlags; ++ assert( (mFlags&(MEM_Int|MEM_IntReal))==0 || pMem->u.i==pX->u.i ); + + /* pMem is the register that is changing. But also mark pX as + ** undefined so that we can quickly detect the shallow-copy error */ +@@ -75361,7 +76616,6 @@ + } + #endif /* SQLITE_DEBUG */ + +- + /* + ** Make an shallow copy of pFrom into pTo. Prior contents of + ** pTo are freed. The pFrom->z field is not duplicated. If +@@ -75507,10 +76761,19 @@ + + pMem->n = nByte; + pMem->flags = flags; +- pMem->enc = (enc==0 ? SQLITE_UTF8 : enc); ++ if( enc ){ ++ pMem->enc = enc; ++#ifdef SQLITE_ENABLE_SESSION ++ }else if( pMem->db==0 ){ ++ pMem->enc = SQLITE_UTF8; ++#endif ++ }else{ ++ assert( pMem->db!=0 ); ++ pMem->enc = ENC(pMem->db); ++ } + + #ifndef SQLITE_OMIT_UTF16 +- if( pMem->enc!=SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){ ++ if( enc>SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){ + return SQLITE_NOMEM_BKPT; + } + #endif +@@ -75537,7 +76800,7 @@ + ** If this routine fails for any reason (malloc returns NULL or unable + ** to read from the disk) then the pMem is left in an inconsistent state. + */ +-static SQLITE_NOINLINE int vdbeMemFromBtreeResize( ++SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( + BtCursor *pCur, /* Cursor pointing at record to retrieve. */ + u32 offset, /* Offset from the start of data to return bytes from. */ + u32 amt, /* Number of bytes to return. */ +@@ -75560,13 +76823,11 @@ + } + return rc; + } +-SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( ++SQLITE_PRIVATE int sqlite3VdbeMemFromBtreeZeroOffset( + BtCursor *pCur, /* Cursor pointing at record to retrieve. */ +- u32 offset, /* Offset from the start of data to return bytes from. */ + u32 amt, /* Number of bytes to return. */ + Mem *pMem /* OUT: Return data in this Mem structure. */ + ){ +- char *zData; /* Data from the btree layer */ + u32 available = 0; /* Number of bytes available on the local btree page */ + int rc = SQLITE_OK; /* Return code */ + +@@ -75576,15 +76837,14 @@ + /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() + ** that both the BtShared and database handle mutexes are held. */ + assert( !sqlite3VdbeMemIsRowSet(pMem) ); +- zData = (char *)sqlite3BtreePayloadFetch(pCur, &available); +- assert( zData!=0 ); ++ pMem->z = (char *)sqlite3BtreePayloadFetch(pCur, &available); ++ assert( pMem->z!=0 ); + +- if( offset+amt<=available ){ +- pMem->z = &zData[offset]; ++ if( amt<=available ){ + pMem->flags = MEM_Blob|MEM_Ephem; + pMem->n = (int)amt; + }else{ +- rc = vdbeMemFromBtreeResize(pCur, offset, amt, pMem); ++ rc = sqlite3VdbeMemFromBtree(pCur, 0, amt, pMem); + } + + return rc; +@@ -75621,7 +76881,7 @@ + assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0 + || pVal->db->mallocFailed ); + if( pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) ){ +- assert( sqlite3VdbeMemConsistentDualRep(pVal) ); ++ assert( sqlite3VdbeMemValidStrRep(pVal) ); + return pVal->z; + }else{ + return 0; +@@ -75644,7 +76904,7 @@ + assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); + assert( !sqlite3VdbeMemIsRowSet(pVal) ); + if( (pVal->flags&(MEM_Str|MEM_Term))==(MEM_Str|MEM_Term) && pVal->enc==enc ){ +- assert( sqlite3VdbeMemConsistentDualRep(pVal) ); ++ assert( sqlite3VdbeMemValidStrRep(pVal) ); + return pVal->z; + } + if( pVal->flags&MEM_Null ){ +@@ -75688,7 +76948,7 @@ + ** an sqlite3_value within the UnpackedRecord.a[] array. + */ + static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ +-#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 ++#ifdef SQLITE_ENABLE_STAT4 + if( p ){ + UnpackedRecord *pRec = p->ppRec[0]; + +@@ -75724,7 +76984,7 @@ + } + #else + UNUSED_PARAMETER(p); +-#endif /* defined(SQLITE_ENABLE_STAT3_OR_STAT4) */ ++#endif /* defined(SQLITE_ENABLE_STAT4) */ + return sqlite3ValueNew(db); + } + +@@ -75748,7 +77008,7 @@ + ** and sets (*ppVal) to NULL. Or, if an error occurs, (*ppVal) is set to + ** NULL and an SQLite error code returned. + */ +-#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 ++#ifdef SQLITE_ENABLE_STAT4 + static int valueFromFunction( + sqlite3 *db, /* The database connection */ + Expr *p, /* The expression to evaluate */ +@@ -75831,7 +77091,7 @@ + } + #else + # define valueFromFunction(a,b,c,d,e,f) SQLITE_OK +-#endif /* defined(SQLITE_ENABLE_STAT3_OR_STAT4) */ ++#endif /* defined(SQLITE_ENABLE_STAT4) */ + + /* + ** Extract a value from the supplied expression in the manner described +@@ -75860,7 +77120,7 @@ + + assert( pExpr!=0 ); + while( (op = pExpr->op)==TK_UPLUS || op==TK_SPAN ) pExpr = pExpr->pLeft; +-#if defined(SQLITE_ENABLE_STAT3_OR_STAT4) ++#if defined(SQLITE_ENABLE_STAT4) + if( op==TK_REGISTER ) op = pExpr->op2; + #else + if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; +@@ -75909,7 +77169,12 @@ + }else{ + sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); + } +- if( pVal->flags & (MEM_Int|MEM_Real) ) pVal->flags &= ~MEM_Str; ++ assert( (pVal->flags & MEM_IntReal)==0 ); ++ if( pVal->flags & (MEM_Int|MEM_IntReal|MEM_Real) ){ ++ testcase( pVal->flags & MEM_Int ); ++ testcase( pVal->flags & MEM_Real ); ++ pVal->flags &= ~MEM_Str; ++ } + if( enc!=SQLITE_UTF8 ){ + rc = sqlite3VdbeChangeEncoding(pVal, enc); + } +@@ -75922,7 +77187,11 @@ + if( pVal->flags & MEM_Real ){ + pVal->u.r = -pVal->u.r; + }else if( pVal->u.i==SMALLEST_INT64 ){ ++#ifndef SQLITE_OMIT_FLOATING_POINT + pVal->u.r = -(double)SMALLEST_INT64; ++#else ++ pVal->u.r = LARGEST_INT64; ++#endif + MemSetTypeFlag(pVal, MEM_Real); + }else{ + pVal->u.i = -pVal->u.i; +@@ -75932,7 +77201,7 @@ + }else if( op==TK_NULL ){ + pVal = valueNew(db, pCtx); + if( pVal==0 ) goto no_mem; +- sqlite3VdbeMemNumerify(pVal); ++ sqlite3VdbeMemSetNull(pVal); + } + #ifndef SQLITE_OMIT_BLOB_LITERAL + else if( op==TK_BLOB ){ +@@ -75948,7 +77217,7 @@ + 0, SQLITE_DYNAMIC); + } + #endif +-#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 ++#ifdef SQLITE_ENABLE_STAT4 + else if( op==TK_FUNCTION && pCtx!=0 ){ + rc = valueFromFunction(db, pExpr, enc, affinity, &pVal, pCtx); + } +@@ -75965,13 +77234,13 @@ + return rc; + + no_mem: +-#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 ++#ifdef SQLITE_ENABLE_STAT4 + if( pCtx==0 || pCtx->pParse->nErr==0 ) + #endif + sqlite3OomFault(db); + sqlite3DbFree(db, zVal); + assert( *ppVal==0 ); +-#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 ++#ifdef SQLITE_ENABLE_STAT4 + if( pCtx==0 ) sqlite3ValueFree(pVal); + #else + assert( pCtx==0 ); sqlite3ValueFree(pVal); +@@ -75999,56 +77268,7 @@ + return pExpr ? valueFromExpr(db, pExpr, enc, affinity, ppVal, 0) : 0; + } + +-#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +-/* +-** The implementation of the sqlite_record() function. This function accepts +-** a single argument of any type. The return value is a formatted database +-** record (a blob) containing the argument value. +-** +-** This is used to convert the value stored in the 'sample' column of the +-** sqlite_stat3 table to the record format SQLite uses internally. +-*/ +-static void recordFunc( +- sqlite3_context *context, +- int argc, +- sqlite3_value **argv +-){ +- const int file_format = 1; +- u32 iSerial; /* Serial type */ +- int nSerial; /* Bytes of space for iSerial as varint */ +- u32 nVal; /* Bytes of space required for argv[0] */ +- int nRet; +- sqlite3 *db; +- u8 *aRet; +- +- UNUSED_PARAMETER( argc ); +- iSerial = sqlite3VdbeSerialType(argv[0], file_format, &nVal); +- nSerial = sqlite3VarintLen(iSerial); +- db = sqlite3_context_db_handle(context); +- +- nRet = 1 + nSerial + nVal; +- aRet = sqlite3DbMallocRawNN(db, nRet); +- if( aRet==0 ){ +- sqlite3_result_error_nomem(context); +- }else{ +- aRet[0] = nSerial+1; +- putVarint32(&aRet[1], iSerial); +- sqlite3VdbeSerialPut(&aRet[1+nSerial], argv[0], iSerial); +- sqlite3_result_blob(context, aRet, nRet, SQLITE_TRANSIENT); +- sqlite3DbFreeNN(db, aRet); +- } +-} +- +-/* +-** Register built-in functions used to help read ANALYZE data. +-*/ +-SQLITE_PRIVATE void sqlite3AnalyzeFunctions(void){ +- static FuncDef aAnalyzeTableFuncs[] = { +- FUNCTION(sqlite_record, 1, 0, 0, recordFunc), +- }; +- sqlite3InsertBuiltinFuncs(aAnalyzeTableFuncs, ArraySize(aAnalyzeTableFuncs)); +-} +- ++#ifdef SQLITE_ENABLE_STAT4 + /* + ** Attempt to extract a value from pExpr and use it to construct *ppVal. + ** +@@ -76330,6 +77550,10 @@ + /* #include "sqliteInt.h" */ + /* #include "vdbeInt.h" */ + ++/* Forward references */ ++static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef); ++static void vdbeFreeOpArray(sqlite3 *, Op *, int); ++ + /* + ** Create a new virtual database engine. + */ +@@ -76357,6 +77581,13 @@ + return p; + } + ++/* ++** Return the Parse object that owns a Vdbe object. ++*/ ++SQLITE_PRIVATE Parse *sqlite3VdbeParser(Vdbe *p){ ++ return p->pParse; ++} ++ + /* + ** Change the error string stored in Vdbe.zErrMsg + */ +@@ -76437,7 +77668,7 @@ + zTmp = pA->zSql; + pA->zSql = pB->zSql; + pB->zSql = zTmp; +-#if 0 ++#ifdef SQLITE_ENABLE_NORMALIZE + zTmp = pA->zNormSql; + pA->zNormSql = pB->zNormSql; + pB->zNormSql = zTmp; +@@ -76498,9 +77729,16 @@ + #ifdef SQLITE_DEBUG + /* This routine is just a convenient place to set a breakpoint that will + ** fire after each opcode is inserted and displayed using +-** "PRAGMA vdbe_addoptrace=on". ++** "PRAGMA vdbe_addoptrace=on". Parameters "pc" (program counter) and ++** pOp are available to make the breakpoint conditional. ++** ++** Other useful labels for breakpoints include: ++** test_trace_breakpoint(pc,pOp) ++** sqlite3CorruptError(lineno) ++** sqlite3MisuseError(lineno) ++** sqlite3CantopenError(lineno) + */ +-static void test_addop_breakpoint(void){ ++static void test_addop_breakpoint(int pc, Op *pOp){ + static int n = 0; + n++; + } +@@ -76553,7 +77791,7 @@ + #ifdef SQLITE_DEBUG + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + sqlite3VdbePrintOp(0, i, &p->aOp[i]); +- test_addop_breakpoint(); ++ test_addop_breakpoint(i, &p->aOp[i]); + } + #endif + #ifdef VDBE_PROFILE +@@ -76636,6 +77874,49 @@ + return addr; + } + ++/* ++** Add an OP_Function or OP_PureFunc opcode. ++** ++** The eCallCtx argument is information (typically taken from Expr.op2) ++** that describes the calling context of the function. 0 means a general ++** function call. NC_IsCheck means called by a check constraint, ++** NC_IdxExpr means called as part of an index expression. NC_PartIdx ++** means in the WHERE clause of a partial index. NC_GenCol means called ++** while computing a generated column value. 0 is the usual case. ++*/ ++SQLITE_PRIVATE int sqlite3VdbeAddFunctionCall( ++ Parse *pParse, /* Parsing context */ ++ int p1, /* Constant argument mask */ ++ int p2, /* First argument register */ ++ int p3, /* Register into which results are written */ ++ int nArg, /* Number of argument */ ++ const FuncDef *pFunc, /* The function to be invoked */ ++ int eCallCtx /* Calling context */ ++){ ++ Vdbe *v = pParse->pVdbe; ++ int nByte; ++ int addr; ++ sqlite3_context *pCtx; ++ assert( v ); ++ nByte = sizeof(*pCtx) + (nArg-1)*sizeof(sqlite3_value*); ++ pCtx = sqlite3DbMallocRawNN(pParse->db, nByte); ++ if( pCtx==0 ){ ++ assert( pParse->db->mallocFailed ); ++ freeEphemeralFunction(pParse->db, (FuncDef*)pFunc); ++ return 0; ++ } ++ pCtx->pOut = 0; ++ pCtx->pFunc = (FuncDef*)pFunc; ++ pCtx->pVdbe = 0; ++ pCtx->isError = 0; ++ pCtx->argc = nArg; ++ pCtx->iOp = sqlite3VdbeCurrentAddr(v); ++ addr = sqlite3VdbeAddOp4(v, eCallCtx ? OP_PureFunc : OP_Function, ++ p1, p2, p3, (char*)pCtx, P4_FUNCCTX); ++ sqlite3VdbeChangeP5(v, eCallCtx & NC_SelfRef); ++ return addr; ++} ++ + /* + ** Add an opcode that includes the p4 value with a P4_INT64 or + ** P4_REAL type. +@@ -76678,7 +77959,7 @@ + #endif + + /* +-** Add a new OP_ opcode. ++** Add a new OP_Explain opcode. + ** + ** If the bPush flag is true, then make this opcode the parent for + ** subsequent Explains until sqlite3VdbeExplainPop() is called. +@@ -76928,6 +78209,7 @@ + ** * OP_HaltIfNull with P1=SQLITE_CONSTRAINT and P2=OE_Abort. + ** * OP_Destroy + ** * OP_VUpdate ++** * OP_VCreate + ** * OP_VRename + ** * OP_FkCounter with P2==0 (immediate foreign key constraint) + ** * OP_CreateBtree/BTREE_INTKEY and OP_InitCoroutine +@@ -76944,6 +78226,7 @@ + int hasAbort = 0; + int hasFkCounter = 0; + int hasCreateTable = 0; ++ int hasCreateIndex = 0; + int hasInitCoroutine = 0; + Op *pOp; + VdbeOpIter sIter; +@@ -76954,7 +78237,8 @@ + int opcode = pOp->opcode; + if( opcode==OP_Destroy || opcode==OP_VUpdate || opcode==OP_VRename + || opcode==OP_VDestroy +- || (opcode==OP_Function0 && pOp->p4.pFunc->funcFlags&SQLITE_FUNC_INTERNAL) ++ || opcode==OP_VCreate ++ || (opcode==OP_ParseSchema && pOp->p4.z==0) + || ((opcode==OP_Halt || opcode==OP_HaltIfNull) + && ((pOp->p1)!=SQLITE_OK && pOp->p2==OE_Abort)) + ){ +@@ -76962,6 +78246,14 @@ + break; + } + if( opcode==OP_CreateBtree && pOp->p3==BTREE_INTKEY ) hasCreateTable = 1; ++ if( mayAbort ){ ++ /* hasCreateIndex may also be set for some DELETE statements that use ++ ** OP_Clear. So this routine may end up returning true in the case ++ ** where a "DELETE FROM tbl" has a statement-journal but does not ++ ** require one. This is not so bad - it is an inefficiency, not a bug. */ ++ if( opcode==OP_CreateBtree && pOp->p3==BTREE_BLOBKEY ) hasCreateIndex = 1; ++ if( opcode==OP_Clear ) hasCreateIndex = 1; ++ } + if( opcode==OP_InitCoroutine ) hasInitCoroutine = 1; + #ifndef SQLITE_OMIT_FOREIGN_KEY + if( opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1 ){ +@@ -76977,7 +78269,8 @@ + ** true for this case to prevent the assert() in the callers frame + ** from failing. */ + return ( v->db->mallocFailed || hasAbort==mayAbort || hasFkCounter +- || (hasCreateTable && hasInitCoroutine) ); ++ || (hasCreateTable && hasInitCoroutine) || hasCreateIndex ++ ); + } + #endif /* SQLITE_DEBUG - the sqlite3AssertMayAbort() function */ + +@@ -77282,16 +78575,16 @@ + ** Change the value of the opcode, or P1, P2, P3, or P5 operands + ** for a specific instruction. + */ +-SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe *p, u32 addr, u8 iNewOpcode){ ++SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe *p, int addr, u8 iNewOpcode){ + sqlite3VdbeGetOp(p,addr)->opcode = iNewOpcode; + } +-SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, u32 addr, int val){ ++SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, int addr, int val){ + sqlite3VdbeGetOp(p,addr)->p1 = val; + } +-SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, u32 addr, int val){ ++SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, int addr, int val){ + sqlite3VdbeGetOp(p,addr)->p2 = val; + } +-SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){ ++SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, int addr, int val){ + sqlite3VdbeGetOp(p,addr)->p3 = val; + } + SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){ +@@ -77307,6 +78600,34 @@ + sqlite3VdbeChangeP2(p, addr, p->nOp); + } + ++/* ++** Change the P2 operand of the jump instruction at addr so that ++** the jump lands on the next opcode. Or if the jump instruction was ++** the previous opcode (and is thus a no-op) then simply back up ++** the next instruction counter by one slot so that the jump is ++** overwritten by the next inserted opcode. ++** ++** This routine is an optimization of sqlite3VdbeJumpHere() that ++** strives to omit useless byte-code like this: ++** ++** 7 Once 0 8 0 ++** 8 ... ++*/ ++SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe *p, int addr){ ++ if( addr==p->nOp-1 ){ ++ assert( p->aOp[addr].opcode==OP_Once ++ || p->aOp[addr].opcode==OP_If ++ || p->aOp[addr].opcode==OP_FkIfZero ); ++ assert( p->aOp[addr].p4type==0 ); ++#ifdef SQLITE_VDBE_COVERAGE ++ sqlite3VdbeGetOp(p,-1)->iSrcLine = 0; /* Erase VdbeCoverage() macros */ ++#endif ++ p->nOp--; ++ }else{ ++ sqlite3VdbeChangeP2(p, addr, p->nOp); ++ } ++} ++ + + /* + ** If the input FuncDef structure is ephemeral, then free it. If +@@ -77318,8 +78639,6 @@ + } + } + +-static void vdbeFreeOpArray(sqlite3 *, Op *, int); +- + /* + ** Delete a P4 value if necessary. + */ +@@ -77329,7 +78648,7 @@ + } + static SQLITE_NOINLINE void freeP4FuncCtx(sqlite3 *db, sqlite3_context *p){ + freeEphemeralFunction(db, p->pFunc); +- sqlite3DbFreeNN(db, p); ++ sqlite3DbFreeNN(db, p); + } + static void freeP4(sqlite3 *db, int p4type, void *p4){ + assert( db ); +@@ -77403,6 +78722,13 @@ + pVdbe->pProgram = p; + } + ++/* ++** Return true if the given Vdbe has any SubPrograms. ++*/ ++SQLITE_PRIVATE int sqlite3VdbeHasSubProgram(Vdbe *pVdbe){ ++ return pVdbe->pProgram!=0; ++} ++ + /* + ** Change the opcode at addr into OP_Noop + */ +@@ -77430,6 +78756,41 @@ + } + } + ++#ifdef SQLITE_DEBUG ++/* ++** Generate an OP_ReleaseReg opcode to indicate that a range of ++** registers, except any identified by mask, are no longer in use. ++*/ ++SQLITE_PRIVATE void sqlite3VdbeReleaseRegisters( ++ Parse *pParse, /* Parsing context */ ++ int iFirst, /* Index of first register to be released */ ++ int N, /* Number of registers to release */ ++ u32 mask, /* Mask of registers to NOT release */ ++ int bUndefine /* If true, mark registers as undefined */ ++){ ++ if( N==0 ) return; ++ assert( pParse->pVdbe ); ++ assert( iFirst>=1 ); ++ assert( iFirst+N-1<=pParse->nMem ); ++ if( N<=31 && mask!=0 ){ ++ while( N>0 && (mask&1)!=0 ){ ++ mask >>= 1; ++ iFirst++; ++ N--; ++ } ++ while( N>0 && N<=32 && (mask & MASKBIT32(N-1))!=0 ){ ++ mask &= ~MASKBIT32(N-1); ++ N--; ++ } ++ } ++ if( N>0 ){ ++ sqlite3VdbeAddOp3(pParse->pVdbe, OP_ReleaseReg, iFirst, N, *(int*)&mask); ++ if( bUndefine ) sqlite3VdbeChangeP5(pParse->pVdbe, 1); ++ } ++} ++#endif /* SQLITE_DEBUG */ ++ ++ + /* + ** Change the value of the P4 operand for a specific instruction. + ** This routine is useful when a large program is loaded from a +@@ -77547,7 +78908,8 @@ + */ + static void vdbeVComment(Vdbe *p, const char *zFormat, va_list ap){ + assert( p->nOp>0 || p->aOp==0 ); +- assert( p->aOp==0 || p->aOp[p->nOp-1].zComment==0 || p->db->mallocFailed ); ++ assert( p->aOp==0 || p->aOp[p->nOp-1].zComment==0 || p->db->mallocFailed ++ || p->pParse->nErr>0 ); + if( p->nOp ){ + assert( p->aOp ); + sqlite3DbFree(p->db, p->aOp[p->nOp-1].zComment); +@@ -77637,17 +78999,19 @@ + ** "PX@PY+1" -> "r[X..X+Y]" or "r[x]" if y is 0 + ** "PY..PY" -> "r[X..Y]" or "r[x]" if y<=x + */ +-static int displayComment( ++SQLITE_PRIVATE char *sqlite3VdbeDisplayComment( ++ sqlite3 *db, /* Optional - Oom error reporting only */ + const Op *pOp, /* The opcode to be commented */ +- const char *zP4, /* Previously obtained value for P4 */ +- char *zTemp, /* Write result here */ +- int nTemp /* Space available in zTemp[] */ ++ const char *zP4 /* Previously obtained value for P4 */ + ){ + const char *zOpName; + const char *zSynopsis; + int nOpName; +- int ii, jj; ++ int ii; + char zAlt[50]; ++ StrAccum x; ++ ++ sqlite3StrAccumInit(&x, 0, 0, 0, SQLITE_MAX_LENGTH); + zOpName = sqlite3OpcodeName(pOp->opcode); + nOpName = sqlite3Strlen30(zOpName); + if( zOpName[nOpName+1] ){ +@@ -77662,53 +79026,64 @@ + } + zSynopsis = zAlt; + } +- for(ii=jj=0; jj<nTemp-1 && (c = zSynopsis[ii])!=0; ii++){ ++ for(ii=0; (c = zSynopsis[ii])!=0; ii++){ + if( c=='P' ){ + c = zSynopsis[++ii]; + if( c=='4' ){ +- sqlite3_snprintf(nTemp-jj, zTemp+jj, "%s", zP4); ++ sqlite3_str_appendall(&x, zP4); + }else if( c=='X' ){ +- sqlite3_snprintf(nTemp-jj, zTemp+jj, "%s", pOp->zComment); ++ sqlite3_str_appendall(&x, pOp->zComment); + seenCom = 1; + }else{ + int v1 = translateP(c, pOp); + int v2; +- sqlite3_snprintf(nTemp-jj, zTemp+jj, "%d", v1); + if( strncmp(zSynopsis+ii+1, "@P", 2)==0 ){ + ii += 3; +- jj += sqlite3Strlen30(zTemp+jj); + v2 = translateP(zSynopsis[ii], pOp); + if( strncmp(zSynopsis+ii+1,"+1",2)==0 ){ + ii += 2; + v2++; + } +- if( v2>1 ){ +- sqlite3_snprintf(nTemp-jj, zTemp+jj, "..%d", v1+v2-1); ++ if( v2<2 ){ ++ sqlite3_str_appendf(&x, "%d", v1); ++ }else{ ++ sqlite3_str_appendf(&x, "%d..%d", v1, v1+v2-1); ++ } ++ }else if( strncmp(zSynopsis+ii+1, "@NP", 3)==0 ){ ++ sqlite3_context *pCtx = pOp->p4.pCtx; ++ if( pOp->p4type!=P4_FUNCCTX || pCtx->argc==1 ){ ++ sqlite3_str_appendf(&x, "%d", v1); ++ }else if( pCtx->argc>1 ){ ++ sqlite3_str_appendf(&x, "%d..%d", v1, v1+pCtx->argc-1); ++ }else{ ++ assert( x.nChar>2 ); ++ x.nChar -= 2; ++ ii++; ++ } ++ ii += 3; ++ }else{ ++ sqlite3_str_appendf(&x, "%d", v1); ++ if( strncmp(zSynopsis+ii+1, "..P3", 4)==0 && pOp->p3==0 ){ ++ ii += 4; + } +- }else if( strncmp(zSynopsis+ii+1, "..P3", 4)==0 && pOp->p3==0 ){ +- ii += 4; + } + } +- jj += sqlite3Strlen30(zTemp+jj); + }else{ +- zTemp[jj++] = c; ++ sqlite3_str_appendchar(&x, 1, c); + } + } +- if( !seenCom && jj<nTemp-5 && pOp->zComment ){ +- sqlite3_snprintf(nTemp-jj, zTemp+jj, "; %s", pOp->zComment); +- jj += sqlite3Strlen30(zTemp+jj); ++ if( !seenCom && pOp->zComment ){ ++ sqlite3_str_appendf(&x, "; %s", pOp->zComment); + } +- if( jj<nTemp ) zTemp[jj] = 0; + }else if( pOp->zComment ){ +- sqlite3_snprintf(nTemp, zTemp, "%s", pOp->zComment); +- jj = sqlite3Strlen30(zTemp); +- }else{ +- zTemp[0] = 0; +- jj = 0; ++ sqlite3_str_appendall(&x, pOp->zComment); ++ } ++ if( (x.accError & SQLITE_NOMEM)!=0 && db!=0 ){ ++ sqlite3OomFault(db); + } +- return jj; ++ return sqlite3StrAccumFinish(&x); + } +-#endif /* SQLITE_DEBUG */ ++#endif /* SQLITE_ENABLE_EXPLAIN_COMMENTS */ + + #if VDBE_DISPLAY_P4 && defined(SQLITE_ENABLE_CURSOR_HINTS) + /* +@@ -77789,23 +79164,25 @@ + ** Compute a string that describes the P4 parameter for an opcode. + ** Use zTemp for any required temporary buffer space. + */ +-static char *displayP4(Op *pOp, char *zTemp, int nTemp){ +- char *zP4 = zTemp; ++SQLITE_PRIVATE char *sqlite3VdbeDisplayP4(sqlite3 *db, Op *pOp){ ++ char *zP4 = 0; + StrAccum x; +- assert( nTemp>=20 ); +- sqlite3StrAccumInit(&x, 0, zTemp, nTemp, 0); ++ ++ sqlite3StrAccumInit(&x, 0, 0, 0, SQLITE_MAX_LENGTH); + switch( pOp->p4type ){ + case P4_KEYINFO: { + int j; + KeyInfo *pKeyInfo = pOp->p4.pKeyInfo; +- assert( pKeyInfo->aSortOrder!=0 ); ++ assert( pKeyInfo->aSortFlags!=0 ); + sqlite3_str_appendf(&x, "k(%d", pKeyInfo->nKeyField); + for(j=0; j<pKeyInfo->nKeyField; j++){ + CollSeq *pColl = pKeyInfo->aColl[j]; + const char *zColl = pColl ? pColl->zName : ""; + if( strcmp(zColl, "BINARY")==0 ) zColl = "B"; +- sqlite3_str_appendf(&x, ",%s%s", +- pKeyInfo->aSortOrder[j] ? "-" : "", zColl); ++ sqlite3_str_appendf(&x, ",%s%s%s", ++ (pKeyInfo->aSortFlags[j] & KEYINFO_ORDER_DESC) ? "-" : "", ++ (pKeyInfo->aSortFlags[j] & KEYINFO_ORDER_BIGNULL)? "N." : "", ++ zColl); + } + sqlite3_str_append(&x, ")", 1); + break; +@@ -77817,8 +79194,11 @@ + } + #endif + case P4_COLLSEQ: { ++ static const char *const encnames[] = {"?", "8", "16LE", "16BE"}; + CollSeq *pColl = pOp->p4.pColl; +- sqlite3_str_appendf(&x, "(%.20s)", pColl->zName); ++ assert( pColl->enc>=0 && pColl->enc<4 ); ++ sqlite3_str_appendf(&x, "%.18s-%s", pColl->zName, ++ encnames[pColl->enc]); + break; + } + case P4_FUNCDEF: { +@@ -77826,13 +79206,11 @@ + sqlite3_str_appendf(&x, "%s(%d)", pDef->zName, pDef->nArg); + break; + } +-#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) + case P4_FUNCCTX: { + FuncDef *pDef = pOp->p4.pCtx->pFunc; + sqlite3_str_appendf(&x, "%s(%d)", pDef->zName, pDef->nArg); + break; + } +-#endif + case P4_INT64: { + sqlite3_str_appendf(&x, "%lld", *pOp->p4.pI64); + break; +@@ -77849,7 +79227,7 @@ + Mem *pMem = pOp->p4.pMem; + if( pMem->flags & MEM_Str ){ + zP4 = pMem->z; +- }else if( pMem->flags & MEM_Int ){ ++ }else if( pMem->flags & (MEM_Int|MEM_IntReal) ){ + sqlite3_str_appendf(&x, "%lld", pMem->u.i); + }else if( pMem->flags & MEM_Real ){ + sqlite3_str_appendf(&x, "%.16g", pMem->u.r); +@@ -77874,36 +79252,32 @@ + int n = ai[0]; /* The first element of an INTARRAY is always the + ** count of the number of elements to follow */ + for(i=1; i<=n; i++){ +- sqlite3_str_appendf(&x, ",%d", ai[i]); ++ sqlite3_str_appendf(&x, "%c%d", (i==1 ? '[' : ','), ai[i]); + } +- zTemp[0] = '['; + sqlite3_str_append(&x, "]", 1); + break; + } + case P4_SUBPROGRAM: { +- sqlite3_str_appendf(&x, "program"); ++ zP4 = "program"; + break; + } + case P4_DYNBLOB: + case P4_ADVANCE: { +- zTemp[0] = 0; + break; + } + case P4_TABLE: { +- sqlite3_str_appendf(&x, "%s", pOp->p4.pTab->zName); ++ zP4 = pOp->p4.pTab->zName; + break; + } + default: { + zP4 = pOp->p4.z; +- if( zP4==0 ){ +- zP4 = zTemp; +- zTemp[0] = 0; +- } + } + } +- sqlite3StrAccumFinish(&x); +- assert( zP4!=0 ); +- return zP4; ++ if( zP4 ) sqlite3_str_appendall(&x, zP4); ++ if( (x.accError & SQLITE_NOMEM)!=0 ){ ++ sqlite3OomFault(db); ++ } ++ return sqlite3StrAccumFinish(&x); + } + #endif /* VDBE_DISPLAY_P4 */ + +@@ -77993,24 +79367,30 @@ + */ + SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE *pOut, int pc, VdbeOp *pOp){ + char *zP4; +- char zPtr[50]; +- char zCom[100]; ++ char *zCom; ++ sqlite3 dummyDb; + static const char *zFormat1 = "%4d %-13s %4d %4d %4d %-13s %.2X %s\n"; + if( pOut==0 ) pOut = stdout; +- zP4 = displayP4(pOp, zPtr, sizeof(zPtr)); ++ sqlite3BeginBenignMalloc(); ++ dummyDb.mallocFailed = 1; ++ zP4 = sqlite3VdbeDisplayP4(&dummyDb, pOp); + #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS +- displayComment(pOp, zP4, zCom, sizeof(zCom)); ++ zCom = sqlite3VdbeDisplayComment(0, pOp, zP4); + #else +- zCom[0] = 0; ++ zCom = 0; + #endif + /* NB: The sqlite3OpcodeName() function is implemented by code created + ** by the mkopcodeh.awk and mkopcodec.awk scripts which extract the + ** information from the vdbe.c source text */ + fprintf(pOut, zFormat1, pc, +- sqlite3OpcodeName(pOp->opcode), pOp->p1, pOp->p2, pOp->p3, zP4, pOp->p5, +- zCom ++ sqlite3OpcodeName(pOp->opcode), pOp->p1, pOp->p2, pOp->p3, ++ zP4 ? zP4 : "", pOp->p5, ++ zCom ? zCom : "" + ); + fflush(pOut); ++ sqlite3_free(zP4); ++ sqlite3_free(zCom); ++ sqlite3EndBenignMalloc(); + } + #endif + +@@ -78101,6 +79481,121 @@ + pFrame->v->pDelFrame = pFrame; + } + ++#if defined(SQLITE_ENABLE_BYTECODE_VTAB) || !defined(SQLITE_OMIT_EXPLAIN) ++/* ++** Locate the next opcode to be displayed in EXPLAIN or EXPLAIN ++** QUERY PLAN output. ++** ++** Return SQLITE_ROW on success. Return SQLITE_DONE if there are no ++** more opcodes to be displayed. ++*/ ++SQLITE_PRIVATE int sqlite3VdbeNextOpcode( ++ Vdbe *p, /* The statement being explained */ ++ Mem *pSub, /* Storage for keeping track of subprogram nesting */ ++ int eMode, /* 0: normal. 1: EQP. 2: TablesUsed */ ++ int *piPc, /* IN/OUT: Current rowid. Overwritten with next rowid */ ++ int *piAddr, /* OUT: Write index into (*paOp)[] here */ ++ Op **paOp /* OUT: Write the opcode array here */ ++){ ++ int nRow; /* Stop when row count reaches this */ ++ int nSub = 0; /* Number of sub-vdbes seen so far */ ++ SubProgram **apSub = 0; /* Array of sub-vdbes */ ++ int i; /* Next instruction address */ ++ int rc = SQLITE_OK; /* Result code */ ++ Op *aOp = 0; /* Opcode array */ ++ int iPc; /* Rowid. Copy of value in *piPc */ ++ ++ /* When the number of output rows reaches nRow, that means the ++ ** listing has finished and sqlite3_step() should return SQLITE_DONE. ++ ** nRow is the sum of the number of rows in the main program, plus ++ ** the sum of the number of rows in all trigger subprograms encountered ++ ** so far. The nRow value will increase as new trigger subprograms are ++ ** encountered, but p->pc will eventually catch up to nRow. ++ */ ++ nRow = p->nOp; ++ if( pSub!=0 ){ ++ if( pSub->flags&MEM_Blob ){ ++ /* pSub is initiallly NULL. It is initialized to a BLOB by ++ ** the P4_SUBPROGRAM processing logic below */ ++ nSub = pSub->n/sizeof(Vdbe*); ++ apSub = (SubProgram **)pSub->z; ++ } ++ for(i=0; i<nSub; i++){ ++ nRow += apSub[i]->nOp; ++ } ++ } ++ iPc = *piPc; ++ while(1){ /* Loop exits via break */ ++ i = iPc++; ++ if( i>=nRow ){ ++ p->rc = SQLITE_OK; ++ rc = SQLITE_DONE; ++ break; ++ } ++ if( i<p->nOp ){ ++ /* The rowid is small enough that we are still in the ++ ** main program. */ ++ aOp = p->aOp; ++ }else{ ++ /* We are currently listing subprograms. Figure out which one and ++ ** pick up the appropriate opcode. */ ++ int j; ++ i -= p->nOp; ++ assert( apSub!=0 ); ++ assert( nSub>0 ); ++ for(j=0; i>=apSub[j]->nOp; j++){ ++ i -= apSub[j]->nOp; ++ assert( i<apSub[j]->nOp || j+1<nSub ); ++ } ++ aOp = apSub[j]->aOp; ++ } ++ ++ /* When an OP_Program opcode is encounter (the only opcode that has ++ ** a P4_SUBPROGRAM argument), expand the size of the array of subprograms ++ ** kept in p->aMem[9].z to hold the new program - assuming this subprogram ++ ** has not already been seen. ++ */ ++ if( pSub!=0 && aOp[i].p4type==P4_SUBPROGRAM ){ ++ int nByte = (nSub+1)*sizeof(SubProgram*); ++ int j; ++ for(j=0; j<nSub; j++){ ++ if( apSub[j]==aOp[i].p4.pProgram ) break; ++ } ++ if( j==nSub ){ ++ p->rc = sqlite3VdbeMemGrow(pSub, nByte, nSub!=0); ++ if( p->rc!=SQLITE_OK ){ ++ rc = SQLITE_ERROR; ++ break; ++ } ++ apSub = (SubProgram **)pSub->z; ++ apSub[nSub++] = aOp[i].p4.pProgram; ++ MemSetTypeFlag(pSub, MEM_Blob); ++ pSub->n = nSub*sizeof(SubProgram*); ++ nRow += aOp[i].p4.pProgram->nOp; ++ } ++ } ++ if( eMode==0 ) break; ++#ifdef SQLITE_ENABLE_BYTECODE_VTAB ++ if( eMode==2 ){ ++ Op *pOp = aOp + i; ++ if( pOp->opcode==OP_OpenRead ) break; ++ if( pOp->opcode==OP_OpenWrite && (pOp->p5 & OPFLAG_P2ISREG)==0 ) break; ++ if( pOp->opcode==OP_ReopenIdx ) break; ++ }else ++#endif ++ { ++ assert( eMode==1 ); ++ if( aOp[i].opcode==OP_Explain ) break; ++ if( aOp[i].opcode==OP_Init && iPc>1 ) break; ++ } ++ } ++ *piPc = iPc; ++ *piAddr = i; ++ *paOp = aOp; ++ return rc; ++} ++#endif /* SQLITE_ENABLE_BYTECODE_VTAB || !SQLITE_OMIT_EXPLAIN */ ++ + + /* + ** Delete a VdbeFrame object and its contents. VdbeFrame objects are +@@ -78141,16 +79636,14 @@ + SQLITE_PRIVATE int sqlite3VdbeList( + Vdbe *p /* The VDBE */ + ){ +- int nRow; /* Stop when row count reaches this */ +- int nSub = 0; /* Number of sub-vdbes seen so far */ +- SubProgram **apSub = 0; /* Array of sub-vdbes */ + Mem *pSub = 0; /* Memory cell hold array of subprogs */ + sqlite3 *db = p->db; /* The database connection */ + int i; /* Loop counter */ + int rc = SQLITE_OK; /* Return code */ + Mem *pMem = &p->aMem[1]; /* First Mem of result set */ + int bListSubprogs = (p->explain==1 || (db->flags & SQLITE_TriggerEQP)!=0); +- Op *pOp = 0; ++ Op *aOp; /* Array of opcodes */ ++ Op *pOp; /* Current opcode */ + + assert( p->explain ); + assert( p->magic==VDBE_MAGIC_RUN ); +@@ -78170,14 +79663,6 @@ + return SQLITE_ERROR; + } + +- /* When the number of output rows reaches nRow, that means the +- ** listing has finished and sqlite3_step() should return SQLITE_DONE. +- ** nRow is the sum of the number of rows in the main program, plus +- ** the sum of the number of rows in all trigger subprograms encountered +- ** so far. The nRow value will increase as new trigger subprograms are +- ** encountered, but p->pc will eventually catch up to nRow. +- */ +- nRow = p->nOp; + if( bListSubprogs ){ + /* The first 8 memory cells are used for the result set. So we will + ** commandeer the 9th cell to use as storage for an array of pointers +@@ -78185,144 +79670,55 @@ + ** cells. */ + assert( p->nMem>9 ); + pSub = &p->aMem[9]; +- if( pSub->flags&MEM_Blob ){ +- /* On the first call to sqlite3_step(), pSub will hold a NULL. It is +- ** initialized to a BLOB by the P4_SUBPROGRAM processing logic below */ +- nSub = pSub->n/sizeof(Vdbe*); +- apSub = (SubProgram **)pSub->z; +- } +- for(i=0; i<nSub; i++){ +- nRow += apSub[i]->nOp; +- } ++ }else{ ++ pSub = 0; + } + +- while(1){ /* Loop exits via break */ +- i = p->pc++; +- if( i>=nRow ){ +- p->rc = SQLITE_OK; +- rc = SQLITE_DONE; +- break; +- } +- if( i<p->nOp ){ +- /* The output line number is small enough that we are still in the +- ** main program. */ +- pOp = &p->aOp[i]; +- }else{ +- /* We are currently listing subprograms. Figure out which one and +- ** pick up the appropriate opcode. */ +- int j; +- i -= p->nOp; +- for(j=0; i>=apSub[j]->nOp; j++){ +- i -= apSub[j]->nOp; +- } +- pOp = &apSub[j]->aOp[i]; +- } +- +- /* When an OP_Program opcode is encounter (the only opcode that has +- ** a P4_SUBPROGRAM argument), expand the size of the array of subprograms +- ** kept in p->aMem[9].z to hold the new program - assuming this subprogram +- ** has not already been seen. +- */ +- if( bListSubprogs && pOp->p4type==P4_SUBPROGRAM ){ +- int nByte = (nSub+1)*sizeof(SubProgram*); +- int j; +- for(j=0; j<nSub; j++){ +- if( apSub[j]==pOp->p4.pProgram ) break; +- } +- if( j==nSub ){ +- p->rc = sqlite3VdbeMemGrow(pSub, nByte, nSub!=0); +- if( p->rc!=SQLITE_OK ){ +- rc = SQLITE_ERROR; +- break; +- } +- apSub = (SubProgram **)pSub->z; +- apSub[nSub++] = pOp->p4.pProgram; +- pSub->flags |= MEM_Blob; +- pSub->n = nSub*sizeof(SubProgram*); +- nRow += pOp->p4.pProgram->nOp; +- } +- } +- if( p->explain<2 ) break; +- if( pOp->opcode==OP_Explain ) break; +- if( pOp->opcode==OP_Init && p->pc>1 ) break; +- } ++ /* Figure out which opcode is next to display */ ++ rc = sqlite3VdbeNextOpcode(p, pSub, p->explain==2, &p->pc, &i, &aOp); + + if( rc==SQLITE_OK ){ +- if( db->u1.isInterrupted ){ ++ pOp = aOp + i; ++ if( AtomicLoad(&db->u1.isInterrupted) ){ + p->rc = SQLITE_INTERRUPT; + rc = SQLITE_ERROR; + sqlite3VdbeError(p, sqlite3ErrStr(p->rc)); + }else{ +- char *zP4; +- if( p->explain==1 ){ +- pMem->flags = MEM_Int; +- pMem->u.i = i; /* Program counter */ +- pMem++; +- +- pMem->flags = MEM_Static|MEM_Str|MEM_Term; +- pMem->z = (char*)sqlite3OpcodeName(pOp->opcode); /* Opcode */ +- assert( pMem->z!=0 ); +- pMem->n = sqlite3Strlen30(pMem->z); +- pMem->enc = SQLITE_UTF8; +- pMem++; +- } +- +- pMem->flags = MEM_Int; +- pMem->u.i = pOp->p1; /* P1 */ +- pMem++; +- +- pMem->flags = MEM_Int; +- pMem->u.i = pOp->p2; /* P2 */ +- pMem++; +- +- pMem->flags = MEM_Int; +- pMem->u.i = pOp->p3; /* P3 */ +- pMem++; +- +- if( sqlite3VdbeMemClearAndResize(pMem, 100) ){ /* P4 */ +- assert( p->db->mallocFailed ); +- return SQLITE_ERROR; +- } +- pMem->flags = MEM_Str|MEM_Term; +- zP4 = displayP4(pOp, pMem->z, pMem->szMalloc); +- if( zP4!=pMem->z ){ +- pMem->n = 0; +- sqlite3VdbeMemSetStr(pMem, zP4, -1, SQLITE_UTF8, 0); ++ char *zP4 = sqlite3VdbeDisplayP4(db, pOp); ++ if( p->explain==2 ){ ++ sqlite3VdbeMemSetInt64(pMem, pOp->p1); ++ sqlite3VdbeMemSetInt64(pMem+1, pOp->p2); ++ sqlite3VdbeMemSetInt64(pMem+2, pOp->p3); ++ sqlite3VdbeMemSetStr(pMem+3, zP4, -1, SQLITE_UTF8, sqlite3_free); ++ p->nResColumn = 4; + }else{ +- assert( pMem->z!=0 ); +- pMem->n = sqlite3Strlen30(pMem->z); +- pMem->enc = SQLITE_UTF8; +- } +- pMem++; +- +- if( p->explain==1 ){ +- if( sqlite3VdbeMemClearAndResize(pMem, 4) ){ +- assert( p->db->mallocFailed ); +- return SQLITE_ERROR; +- } +- pMem->flags = MEM_Str|MEM_Term; +- pMem->n = 2; +- sqlite3_snprintf(3, pMem->z, "%.2x", pOp->p5); /* P5 */ +- pMem->enc = SQLITE_UTF8; +- pMem++; +- ++ sqlite3VdbeMemSetInt64(pMem+0, i); ++ sqlite3VdbeMemSetStr(pMem+1, (char*)sqlite3OpcodeName(pOp->opcode), ++ -1, SQLITE_UTF8, SQLITE_STATIC); ++ sqlite3VdbeMemSetInt64(pMem+2, pOp->p1); ++ sqlite3VdbeMemSetInt64(pMem+3, pOp->p2); ++ sqlite3VdbeMemSetInt64(pMem+4, pOp->p3); ++ /* pMem+5 for p4 is done last */ ++ sqlite3VdbeMemSetInt64(pMem+6, pOp->p5); + #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS +- if( sqlite3VdbeMemClearAndResize(pMem, 500) ){ +- assert( p->db->mallocFailed ); +- return SQLITE_ERROR; ++ { ++ char *zCom = sqlite3VdbeDisplayComment(db, pOp, zP4); ++ sqlite3VdbeMemSetStr(pMem+7, zCom, -1, SQLITE_UTF8, sqlite3_free); + } +- pMem->flags = MEM_Str|MEM_Term; +- pMem->n = displayComment(pOp, zP4, pMem->z, 500); +- pMem->enc = SQLITE_UTF8; + #else +- pMem->flags = MEM_Null; /* Comment */ ++ sqlite3VdbeMemSetNull(pMem+7); + #endif ++ sqlite3VdbeMemSetStr(pMem+5, zP4, -1, SQLITE_UTF8, sqlite3_free); ++ p->nResColumn = 8; ++ } ++ p->pResultSet = pMem; ++ if( db->mallocFailed ){ ++ p->rc = SQLITE_NOMEM; ++ rc = SQLITE_ERROR; ++ }else{ ++ p->rc = SQLITE_OK; ++ rc = SQLITE_ROW; + } +- +- p->nResColumn = 8 - 4*(p->explain-1); +- p->pResultSet = &p->aMem[1]; +- p->rc = SQLITE_OK; +- rc = SQLITE_ROW; + } + } + return rc; +@@ -78523,8 +79919,27 @@ + + resolveP2Values(p, &nArg); + p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort); +- if( pParse->explain && nMem<10 ){ +- nMem = 10; ++ if( pParse->explain ){ ++ static const char * const azColName[] = { ++ "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", ++ "id", "parent", "notused", "detail" ++ }; ++ int iFirst, mx, i; ++ if( nMem<10 ) nMem = 10; ++ p->explain = pParse->explain; ++ if( pParse->explain==2 ){ ++ sqlite3VdbeSetNumCols(p, 4); ++ iFirst = 8; ++ mx = 12; ++ }else{ ++ sqlite3VdbeSetNumCols(p, 8); ++ iFirst = 0; ++ mx = 8; ++ } ++ for(i=iFirst; i<mx; i++){ ++ sqlite3VdbeSetColName(p, i-iFirst, COLNAME_NAME, ++ azColName[i], SQLITE_STATIC); ++ } + } + p->expired = 0; + +@@ -78562,7 +79977,6 @@ + + p->pVList = pParse->pVList; + pParse->pVList = 0; +- p->explain = pParse->explain; + if( db->mallocFailed ){ + p->nVar = 0; + p->nCursor = 0; +@@ -78874,8 +80288,9 @@ + + /* Select a master journal file name */ + nMainFile = sqlite3Strlen30(zMainFile); +- zMaster = sqlite3MPrintf(db, "%s-mjXXXXXX9XXz", zMainFile); ++ zMaster = sqlite3MPrintf(db, "%.4c%s%.16c", 0,zMainFile,0); + if( zMaster==0 ) return SQLITE_NOMEM_BKPT; ++ zMaster += 4; + do { + u32 iRandom; + if( retryCount ){ +@@ -78905,7 +80320,7 @@ + ); + } + if( rc!=SQLITE_OK ){ +- sqlite3DbFree(db, zMaster); ++ sqlite3DbFree(db, zMaster-4); + return rc; + } + +@@ -78928,7 +80343,7 @@ + if( rc!=SQLITE_OK ){ + sqlite3OsCloseFree(pMaster); + sqlite3OsDelete(pVfs, zMaster, 0); +- sqlite3DbFree(db, zMaster); ++ sqlite3DbFree(db, zMaster-4); + return rc; + } + } +@@ -78942,7 +80357,7 @@ + ){ + sqlite3OsCloseFree(pMaster); + sqlite3OsDelete(pVfs, zMaster, 0); +- sqlite3DbFree(db, zMaster); ++ sqlite3DbFree(db, zMaster-4); + return rc; + } + +@@ -78965,7 +80380,7 @@ + sqlite3OsCloseFree(pMaster); + assert( rc!=SQLITE_BUSY ); + if( rc!=SQLITE_OK ){ +- sqlite3DbFree(db, zMaster); ++ sqlite3DbFree(db, zMaster-4); + return rc; + } + +@@ -78974,7 +80389,7 @@ + ** transaction files are deleted. + */ + rc = sqlite3OsDelete(pVfs, zMaster, 1); +- sqlite3DbFree(db, zMaster); ++ sqlite3DbFree(db, zMaster-4); + zMaster = 0; + if( rc ){ + return rc; +@@ -79211,7 +80626,7 @@ + } + + /* Check for immediate foreign key violations. */ +- if( p->rc==SQLITE_OK ){ ++ if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ + sqlite3VdbeCheckFk(p, 0); + } + +@@ -79613,7 +81028,7 @@ + ** carried out. Seek the cursor now. If an error occurs, return + ** the appropriate error code. + */ +-static int SQLITE_NOINLINE handleDeferredMoveto(VdbeCursor *p){ ++SQLITE_PRIVATE int SQLITE_NOINLINE sqlite3VdbeFinishMoveto(VdbeCursor *p){ + int res, rc; + #ifdef SQLITE_TEST + extern int sqlite3_search_count; +@@ -79680,12 +81095,12 @@ + assert( p->eCurType==CURTYPE_BTREE || p->eCurType==CURTYPE_PSEUDO ); + if( p->deferredMoveto ){ + int iMap; +- if( p->aAltMap && (iMap = p->aAltMap[1+*piCol])>0 ){ ++ if( p->aAltMap && (iMap = p->aAltMap[1+*piCol])>0 && !p->nullRow ){ + *pp = p->pAltCursor; + *piCol = iMap - 1; + return SQLITE_OK; + } +- return handleDeferredMoveto(p); ++ return sqlite3VdbeFinishMoveto(p); + } + if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){ + return handleMovedCursor(p); +@@ -79735,8 +81150,17 @@ + ** of SQLite will not understand those serial types. + */ + ++#if 0 /* Inlined into the OP_MakeRecord opcode */ + /* + ** Return the serial-type for the value stored in pMem. ++** ++** This routine might convert a large MEM_IntReal value into MEM_Real. ++** ++** 2019-07-11: The primary user of this subroutine was the OP_MakeRecord ++** opcode in the byte-code engine. But by moving this routine in-line, we ++** can omit some redundant tests and make that opcode a lot faster. So ++** this routine is now only used by the STAT3 logic and STAT3 support has ++** ended. The code is kept here for historical reference only. + */ + SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ + int flags = pMem->flags; +@@ -79747,11 +81171,13 @@ + *pLen = 0; + return 0; + } +- if( flags&MEM_Int ){ ++ if( flags&(MEM_Int|MEM_IntReal) ){ + /* Figure out whether to use 1, 2, 4, 6 or 8 bytes. */ + # define MAX_6BYTE ((((i64)0x00008000)<<32)-1) + i64 i = pMem->u.i; + u64 u; ++ testcase( flags & MEM_Int ); ++ testcase( flags & MEM_IntReal ); + if( i<0 ){ + u = ~i; + }else{ +@@ -79771,6 +81197,15 @@ + if( u<=2147483647 ){ *pLen = 4; return 4; } + if( u<=MAX_6BYTE ){ *pLen = 6; return 5; } + *pLen = 8; ++ if( flags&MEM_IntReal ){ ++ /* If the value is IntReal and is going to take up 8 bytes to store ++ ** as an integer, then we might as well make it an 8-byte floating ++ ** point value */ ++ pMem->u.r = (double)pMem->u.i; ++ pMem->flags &= ~MEM_IntReal; ++ pMem->flags |= MEM_Real; ++ return 7; ++ } + return 6; + } + if( flags&MEM_Real ){ +@@ -79786,6 +81221,7 @@ + *pLen = n; + return ((n*2) + 12 + ((flags&MEM_Str)!=0)); + } ++#endif /* inlined into OP_MakeRecord */ + + /* + ** The sizes for serial types less than 128 +@@ -79944,7 +81380,7 @@ + ** routine so that in most cases the overhead of moving the stack pointer + ** is avoided. + */ +-static u32 SQLITE_NOINLINE serialGet( ++static u32 serialGet( + const unsigned char *buf, /* Buffer to deserialize from */ + u32 serial_type, /* Serial type to deserialize */ + Mem *pMem /* Memory cell to write value into */ +@@ -79976,7 +81412,7 @@ + assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 ); + swapMixedEndianFloat(x); + memcpy(&pMem->u.r, &x, sizeof(x)); +- pMem->flags = sqlite3IsNaN(pMem->u.r) ? MEM_Null : MEM_Real; ++ pMem->flags = IsNaN(x) ? MEM_Null : MEM_Real; + } + return 8; + } +@@ -80094,7 +81530,7 @@ + p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte); + if( !p ) return 0; + p->aMem = (Mem*)&((char*)p)[ROUND8(sizeof(UnpackedRecord))]; +- assert( pKeyInfo->aSortOrder!=0 ); ++ assert( pKeyInfo->aSortFlags!=0 ); + p->pKeyInfo = pKeyInfo; + p->nField = pKeyInfo->nKeyField + 1; + return p; +@@ -80193,7 +81629,7 @@ + if( szHdr1>98307 ) return SQLITE_CORRUPT; + d1 = szHdr1; + assert( pKeyInfo->nAllField>=pPKey2->nField || CORRUPT_DB ); +- assert( pKeyInfo->aSortOrder!=0 ); ++ assert( pKeyInfo->aSortFlags!=0 ); + assert( pKeyInfo->nKeyField>0 ); + assert( idx1<=szHdr1 || CORRUPT_DB ); + do{ +@@ -80224,7 +81660,12 @@ + pKeyInfo->nAllField>i ? pKeyInfo->aColl[i] : 0); + if( rc!=0 ){ + assert( mem1.szMalloc==0 ); /* See comment below */ +- if( pKeyInfo->aSortOrder[i] ){ ++ if( (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_BIGNULL) ++ && ((mem1.flags & MEM_Null) || (pPKey2->aMem[i].flags & MEM_Null)) ++ ){ ++ rc = -rc; ++ } ++ if( pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_DESC ){ + rc = -rc; /* Invert the result for DESC sort order. */ + } + goto debugCompareEnd; +@@ -80426,8 +81867,13 @@ + + /* At least one of the two values is a number + */ +- if( combined_flags&(MEM_Int|MEM_Real) ){ +- if( (f1 & f2 & MEM_Int)!=0 ){ ++ if( combined_flags&(MEM_Int|MEM_Real|MEM_IntReal) ){ ++ testcase( combined_flags & MEM_Int ); ++ testcase( combined_flags & MEM_Real ); ++ testcase( combined_flags & MEM_IntReal ); ++ if( (f1 & f2 & (MEM_Int|MEM_IntReal))!=0 ){ ++ testcase( f1 & f2 & MEM_Int ); ++ testcase( f1 & f2 & MEM_IntReal ); + if( pMem1->u.i < pMem2->u.i ) return -1; + if( pMem1->u.i > pMem2->u.i ) return +1; + return 0; +@@ -80437,15 +81883,23 @@ + if( pMem1->u.r > pMem2->u.r ) return +1; + return 0; + } +- if( (f1&MEM_Int)!=0 ){ ++ if( (f1&(MEM_Int|MEM_IntReal))!=0 ){ ++ testcase( f1 & MEM_Int ); ++ testcase( f1 & MEM_IntReal ); + if( (f2&MEM_Real)!=0 ){ + return sqlite3IntFloatCompare(pMem1->u.i, pMem2->u.r); ++ }else if( (f2&(MEM_Int|MEM_IntReal))!=0 ){ ++ if( pMem1->u.i < pMem2->u.i ) return -1; ++ if( pMem1->u.i > pMem2->u.i ) return +1; ++ return 0; + }else{ + return -1; + } + } + if( (f1&MEM_Real)!=0 ){ +- if( (f2&MEM_Int)!=0 ){ ++ if( (f2&(MEM_Int|MEM_IntReal))!=0 ){ ++ testcase( f2 & MEM_Int ); ++ testcase( f2 & MEM_IntReal ); + return -sqlite3IntFloatCompare(pMem2->u.i, pMem1->u.r); + }else{ + return -1; +@@ -80587,14 +82041,16 @@ + VVA_ONLY( mem1.szMalloc = 0; ) /* Only needed by assert() statements */ + assert( pPKey2->pKeyInfo->nAllField>=pPKey2->nField + || CORRUPT_DB ); +- assert( pPKey2->pKeyInfo->aSortOrder!=0 ); ++ assert( pPKey2->pKeyInfo->aSortFlags!=0 ); + assert( pPKey2->pKeyInfo->nKeyField>0 ); + assert( idx1<=szHdr1 || CORRUPT_DB ); + do{ + u32 serial_type; + + /* RHS is an integer */ +- if( pRhs->flags & MEM_Int ){ ++ if( pRhs->flags & (MEM_Int|MEM_IntReal) ){ ++ testcase( pRhs->flags & MEM_Int ); ++ testcase( pRhs->flags & MEM_IntReal ); + serial_type = aKey1[idx1]; + testcase( serial_type==12 ); + if( serial_type>=10 ){ +@@ -80642,7 +82098,7 @@ + + /* RHS is a string */ + else if( pRhs->flags & MEM_Str ){ +- getVarint32(&aKey1[idx1], serial_type); ++ getVarint32NR(&aKey1[idx1], serial_type); + testcase( serial_type==12 ); + if( serial_type<12 ){ + rc = -1; +@@ -80676,7 +82132,7 @@ + /* RHS is a blob */ + else if( pRhs->flags & MEM_Blob ){ + assert( (pRhs->flags & MEM_Zero)==0 || pRhs->n==0 ); +- getVarint32(&aKey1[idx1], serial_type); ++ getVarint32NR(&aKey1[idx1], serial_type); + testcase( serial_type==12 ); + if( serial_type<12 || (serial_type & 0x01) ){ + rc = -1; +@@ -80708,8 +82164,14 @@ + } + + if( rc!=0 ){ +- if( pPKey2->pKeyInfo->aSortOrder[i] ){ +- rc = -rc; ++ int sortFlags = pPKey2->pKeyInfo->aSortFlags[i]; ++ if( sortFlags ){ ++ if( (sortFlags & KEYINFO_ORDER_BIGNULL)==0 ++ || ((sortFlags & KEYINFO_ORDER_DESC) ++ !=(serial_type==0 || (pRhs->flags&MEM_Null))) ++ ){ ++ rc = -rc; ++ } + } + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, rc) ); + assert( mem1.szMalloc==0 ); /* See comment below */ +@@ -80859,7 +82321,10 @@ + + assert( pPKey2->aMem[0].flags & MEM_Str ); + vdbeAssertFieldCountWithinLimits(nKey1, pKey1, pPKey2->pKeyInfo); +- getVarint32(&aKey1[1], serial_type); ++ serial_type = (u8)(aKey1[1]); ++ if( serial_type >= 0x80 ){ ++ sqlite3GetVarint32(&aKey1[1], (u32*)&serial_type); ++ } + if( serial_type<12 ){ + res = pPKey2->r1; /* (pKey1/nKey1) is a number or a null */ + }else if( !(serial_type & 0x01) ){ +@@ -80877,7 +82342,11 @@ + nCmp = MIN( pPKey2->aMem[0].n, nStr ); + res = memcmp(&aKey1[szHdr], pPKey2->aMem[0].z, nCmp); + +- if( res==0 ){ ++ if( res>0 ){ ++ res = pPKey2->r2; ++ }else if( res<0 ){ ++ res = pPKey2->r1; ++ }else{ + res = nStr - pPKey2->aMem[0].n; + if( res==0 ){ + if( pPKey2->nField>1 ){ +@@ -80891,10 +82360,6 @@ + }else{ + res = pPKey2->r1; + } +- }else if( res>0 ){ +- res = pPKey2->r2; +- }else{ +- res = pPKey2->r1; + } + } + +@@ -80926,7 +82391,10 @@ + ** header size is (12*5 + 1 + 1) bytes. */ + if( p->pKeyInfo->nAllField<=13 ){ + int flags = p->aMem[0].flags; +- if( p->pKeyInfo->aSortOrder[0] ){ ++ if( p->pKeyInfo->aSortFlags[0] ){ ++ if( p->pKeyInfo->aSortFlags[0] & KEYINFO_ORDER_BIGNULL ){ ++ return sqlite3VdbeRecordCompare; ++ } + p->r1 = 1; + p->r2 = -1; + }else{ +@@ -80939,7 +82407,9 @@ + testcase( flags & MEM_Real ); + testcase( flags & MEM_Null ); + testcase( flags & MEM_Blob ); +- if( (flags & (MEM_Real|MEM_Null|MEM_Blob))==0 && p->pKeyInfo->aColl[0]==0 ){ ++ if( (flags & (MEM_Real|MEM_IntReal|MEM_Null|MEM_Blob))==0 ++ && p->pKeyInfo->aColl[0]==0 ++ ){ + assert( flags & MEM_Str ); + return vdbeRecordCompareString; + } +@@ -80975,13 +82445,13 @@ + + /* Read in the complete content of the index entry */ + sqlite3VdbeMemInit(&m, db, 0); +- rc = sqlite3VdbeMemFromBtree(pCur, 0, (u32)nCellKey, &m); ++ rc = sqlite3VdbeMemFromBtreeZeroOffset(pCur, (u32)nCellKey, &m); + if( rc ){ + return rc; + } + + /* The index entry must begin with a header size */ +- (void)getVarint32((u8*)m.z, szHdr); ++ getVarint32NR((u8*)m.z, szHdr); + testcase( szHdr==3 ); + testcase( szHdr==m.n ); + testcase( szHdr>0x7fffffff ); +@@ -80992,7 +82462,7 @@ + + /* The last field of the index should be an integer - the ROWID. + ** Verify that the last entry really is an integer. */ +- (void)getVarint32((u8*)&m.z[szHdr-1], typeRowid); ++ getVarint32NR((u8*)&m.z[szHdr-1], typeRowid); + testcase( typeRowid==1 ); + testcase( typeRowid==2 ); + testcase( typeRowid==3 ); +@@ -81057,7 +82527,7 @@ + return SQLITE_CORRUPT_BKPT; + } + sqlite3VdbeMemInit(&m, db, 0); +- rc = sqlite3VdbeMemFromBtree(pCur, 0, (u32)nCellKey, &m); ++ rc = sqlite3VdbeMemFromBtreeZeroOffset(pCur, (u32)nCellKey, &m); + if( rc ){ + return rc; + } +@@ -81173,13 +82643,25 @@ + ** features such as 'now'. + */ + SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context *pCtx){ +-#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 ++ const VdbeOp *pOp; ++#ifdef SQLITE_ENABLE_STAT4 + if( pCtx->pVdbe==0 ) return 1; + #endif +- if( pCtx->pVdbe->aOp[pCtx->iOp].opcode==OP_PureFunc ){ +- sqlite3_result_error(pCtx, +- "non-deterministic function in index expression or CHECK constraint", +- -1); ++ pOp = pCtx->pVdbe->aOp + pCtx->iOp; ++ if( pOp->opcode==OP_PureFunc ){ ++ const char *zContext; ++ char *zMsg; ++ if( pOp->p5 & NC_IsCheck ){ ++ zContext = "a CHECK constraint"; ++ }else if( pOp->p5 & NC_GenCol ){ ++ zContext = "a generated column"; ++ }else{ ++ zContext = "an index"; ++ } ++ zMsg = sqlite3_mprintf("non-deterministic use of %s() in %s", ++ pCtx->pFunc->zName, zContext); ++ sqlite3_result_error(pCtx, zMsg, -1); ++ sqlite3_free(zMsg); + return 0; + } + return 1; +@@ -81270,7 +82752,7 @@ + preupdate.keyinfo.db = db; + preupdate.keyinfo.enc = ENC(db); + preupdate.keyinfo.nKeyField = pTab->nCol; +- preupdate.keyinfo.aSortOrder = (u8*)&fakeSortOrder; ++ preupdate.keyinfo.aSortFlags = (u8*)&fakeSortOrder; + preupdate.iKey1 = iKey1; + preupdate.iKey2 = iKey2; + preupdate.pTab = pTab; +@@ -81529,39 +83011,86 @@ + */ + SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ + static const u8 aType[] = { +- SQLITE_BLOB, /* 0x00 */ +- SQLITE_NULL, /* 0x01 */ +- SQLITE_TEXT, /* 0x02 */ +- SQLITE_NULL, /* 0x03 */ +- SQLITE_INTEGER, /* 0x04 */ +- SQLITE_NULL, /* 0x05 */ +- SQLITE_INTEGER, /* 0x06 */ +- SQLITE_NULL, /* 0x07 */ +- SQLITE_FLOAT, /* 0x08 */ +- SQLITE_NULL, /* 0x09 */ +- SQLITE_FLOAT, /* 0x0a */ +- SQLITE_NULL, /* 0x0b */ +- SQLITE_INTEGER, /* 0x0c */ +- SQLITE_NULL, /* 0x0d */ +- SQLITE_INTEGER, /* 0x0e */ +- SQLITE_NULL, /* 0x0f */ +- SQLITE_BLOB, /* 0x10 */ +- SQLITE_NULL, /* 0x11 */ +- SQLITE_TEXT, /* 0x12 */ +- SQLITE_NULL, /* 0x13 */ +- SQLITE_INTEGER, /* 0x14 */ +- SQLITE_NULL, /* 0x15 */ +- SQLITE_INTEGER, /* 0x16 */ +- SQLITE_NULL, /* 0x17 */ +- SQLITE_FLOAT, /* 0x18 */ +- SQLITE_NULL, /* 0x19 */ +- SQLITE_FLOAT, /* 0x1a */ +- SQLITE_NULL, /* 0x1b */ +- SQLITE_INTEGER, /* 0x1c */ +- SQLITE_NULL, /* 0x1d */ +- SQLITE_INTEGER, /* 0x1e */ +- SQLITE_NULL, /* 0x1f */ ++ SQLITE_BLOB, /* 0x00 (not possible) */ ++ SQLITE_NULL, /* 0x01 NULL */ ++ SQLITE_TEXT, /* 0x02 TEXT */ ++ SQLITE_NULL, /* 0x03 (not possible) */ ++ SQLITE_INTEGER, /* 0x04 INTEGER */ ++ SQLITE_NULL, /* 0x05 (not possible) */ ++ SQLITE_INTEGER, /* 0x06 INTEGER + TEXT */ ++ SQLITE_NULL, /* 0x07 (not possible) */ ++ SQLITE_FLOAT, /* 0x08 FLOAT */ ++ SQLITE_NULL, /* 0x09 (not possible) */ ++ SQLITE_FLOAT, /* 0x0a FLOAT + TEXT */ ++ SQLITE_NULL, /* 0x0b (not possible) */ ++ SQLITE_INTEGER, /* 0x0c (not possible) */ ++ SQLITE_NULL, /* 0x0d (not possible) */ ++ SQLITE_INTEGER, /* 0x0e (not possible) */ ++ SQLITE_NULL, /* 0x0f (not possible) */ ++ SQLITE_BLOB, /* 0x10 BLOB */ ++ SQLITE_NULL, /* 0x11 (not possible) */ ++ SQLITE_TEXT, /* 0x12 (not possible) */ ++ SQLITE_NULL, /* 0x13 (not possible) */ ++ SQLITE_INTEGER, /* 0x14 INTEGER + BLOB */ ++ SQLITE_NULL, /* 0x15 (not possible) */ ++ SQLITE_INTEGER, /* 0x16 (not possible) */ ++ SQLITE_NULL, /* 0x17 (not possible) */ ++ SQLITE_FLOAT, /* 0x18 FLOAT + BLOB */ ++ SQLITE_NULL, /* 0x19 (not possible) */ ++ SQLITE_FLOAT, /* 0x1a (not possible) */ ++ SQLITE_NULL, /* 0x1b (not possible) */ ++ SQLITE_INTEGER, /* 0x1c (not possible) */ ++ SQLITE_NULL, /* 0x1d (not possible) */ ++ SQLITE_INTEGER, /* 0x1e (not possible) */ ++ SQLITE_NULL, /* 0x1f (not possible) */ ++ SQLITE_FLOAT, /* 0x20 INTREAL */ ++ SQLITE_NULL, /* 0x21 (not possible) */ ++ SQLITE_TEXT, /* 0x22 INTREAL + TEXT */ ++ SQLITE_NULL, /* 0x23 (not possible) */ ++ SQLITE_FLOAT, /* 0x24 (not possible) */ ++ SQLITE_NULL, /* 0x25 (not possible) */ ++ SQLITE_FLOAT, /* 0x26 (not possible) */ ++ SQLITE_NULL, /* 0x27 (not possible) */ ++ SQLITE_FLOAT, /* 0x28 (not possible) */ ++ SQLITE_NULL, /* 0x29 (not possible) */ ++ SQLITE_FLOAT, /* 0x2a (not possible) */ ++ SQLITE_NULL, /* 0x2b (not possible) */ ++ SQLITE_FLOAT, /* 0x2c (not possible) */ ++ SQLITE_NULL, /* 0x2d (not possible) */ ++ SQLITE_FLOAT, /* 0x2e (not possible) */ ++ SQLITE_NULL, /* 0x2f (not possible) */ ++ SQLITE_BLOB, /* 0x30 (not possible) */ ++ SQLITE_NULL, /* 0x31 (not possible) */ ++ SQLITE_TEXT, /* 0x32 (not possible) */ ++ SQLITE_NULL, /* 0x33 (not possible) */ ++ SQLITE_FLOAT, /* 0x34 (not possible) */ ++ SQLITE_NULL, /* 0x35 (not possible) */ ++ SQLITE_FLOAT, /* 0x36 (not possible) */ ++ SQLITE_NULL, /* 0x37 (not possible) */ ++ SQLITE_FLOAT, /* 0x38 (not possible) */ ++ SQLITE_NULL, /* 0x39 (not possible) */ ++ SQLITE_FLOAT, /* 0x3a (not possible) */ ++ SQLITE_NULL, /* 0x3b (not possible) */ ++ SQLITE_FLOAT, /* 0x3c (not possible) */ ++ SQLITE_NULL, /* 0x3d (not possible) */ ++ SQLITE_FLOAT, /* 0x3e (not possible) */ ++ SQLITE_NULL, /* 0x3f (not possible) */ + }; ++#ifdef SQLITE_DEBUG ++ { ++ int eType = SQLITE_BLOB; ++ if( pVal->flags & MEM_Null ){ ++ eType = SQLITE_NULL; ++ }else if( pVal->flags & (MEM_Real|MEM_IntReal) ){ ++ eType = SQLITE_FLOAT; ++ }else if( pVal->flags & MEM_Int ){ ++ eType = SQLITE_INTEGER; ++ }else if( pVal->flags & MEM_Str ){ ++ eType = SQLITE_TEXT; ++ } ++ assert( eType == aType[pVal->flags&MEM_AffMask] ); ++ } ++#endif + return aType[pVal->flags&MEM_AffMask]; + } + +@@ -81811,6 +83340,21 @@ + sqlite3OomFault(pCtx->pOut->db); + } + ++#ifndef SQLITE_UNTESTABLE ++/* Force the INT64 value currently stored as the result to be ++** a MEM_IntReal value. See the SQLITE_TESTCTRL_RESULT_INTREAL ++** test-control. ++*/ ++SQLITE_PRIVATE void sqlite3ResultIntReal(sqlite3_context *pCtx){ ++ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); ++ if( pCtx->pOut->flags & MEM_Int ){ ++ pCtx->pOut->flags &= ~MEM_Int; ++ pCtx->pOut->flags |= MEM_IntReal; ++ } ++} ++#endif ++ ++ + /* + ** This function is called after a transaction has been committed. It + ** invokes callbacks registered with sqlite3_wal_hook() as required. +@@ -81896,7 +83440,7 @@ + ** from interrupting a statement that has not yet started. + */ + if( db->nVdbeActive==0 ){ +- db->u1.isInterrupted = 0; ++ AtomicStore(&db->u1.isInterrupted, 0); + } + + assert( db->nVdbeWrite>0 || db->autoCommit==0 +@@ -82077,7 +83621,7 @@ + */ + SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context *p){ + int rc; +-#ifndef SQLITE_ENABLE_STAT3_OR_STAT4 ++#ifndef SQLITE_ENABLE_STAT4 + sqlite3_int64 *piTime = &p->pVdbe->iCurrentTime; + assert( p->pVdbe!=0 ); + #else +@@ -82142,7 +83686,7 @@ + AuxData *pAuxData; + + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); +-#if SQLITE_ENABLE_STAT3_OR_STAT4 ++#if SQLITE_ENABLE_STAT4 + if( pCtx->pVdbe==0 ) return 0; + #else + assert( pCtx->pVdbe!=0 ); +@@ -82176,7 +83720,7 @@ + Vdbe *pVdbe = pCtx->pVdbe; + + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); +-#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 ++#ifdef SQLITE_ENABLE_STAT4 + if( pVdbe==0 ) goto failed; + #else + assert( pVdbe!=0 ); +@@ -82588,7 +84132,7 @@ + /* If the bit corresponding to this variable in Vdbe.expmask is set, then + ** binding a new value to this variable invalidates the current query plan. + ** +- ** IMPLEMENTATION-OF: R-48440-37595 If the specific value bound to host ++ ** IMPLEMENTATION-OF: R-57496-20354 If the specific value bound to a host + ** parameter in the WHERE clause might influence the choice of query plan + ** for a statement, then the statement will be automatically recompiled, + ** as if there had been a schema change, on the first sqlite3_step() call +@@ -83064,7 +84608,7 @@ + goto preupdate_old_out; + } + if( p->pPk ){ +- iIdx = sqlite3ColumnOfIndex(p->pPk, iIdx); ++ iIdx = sqlite3TableColumnToIndex(p->pPk, iIdx); + } + if( iIdx>=p->pCsr->nField || iIdx<0 ){ + rc = SQLITE_RANGE; +@@ -83097,7 +84641,9 @@ + }else if( iIdx>=p->pUnpacked->nField ){ + *ppValue = (sqlite3_value *)columnNullValue(); + }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){ +- if( pMem->flags & MEM_Int ){ ++ if( pMem->flags & (MEM_Int|MEM_IntReal) ){ ++ testcase( pMem->flags & MEM_Int ); ++ testcase( pMem->flags & MEM_IntReal ); + sqlite3VdbeMemRealify(pMem); + } + } +@@ -83152,7 +84698,7 @@ + goto preupdate_new_out; + } + if( p->pPk && p->op!=SQLITE_UPDATE ){ +- iIdx = sqlite3ColumnOfIndex(p->pPk, iIdx); ++ iIdx = sqlite3TableColumnToIndex(p->pPk, iIdx); + } + if( iIdx>=p->pCsr->nField || iIdx<0 ){ + rc = SQLITE_RANGE; +@@ -83416,7 +84962,7 @@ + pVar = &p->aVar[idx-1]; + if( pVar->flags & MEM_Null ){ + sqlite3_str_append(&out, "NULL", 4); +- }else if( pVar->flags & MEM_Int ){ ++ }else if( pVar->flags & (MEM_Int|MEM_IntReal) ){ + sqlite3_str_appendf(&out, "%lld", pVar->u.i); + }else if( pVar->flags & MEM_Real ){ + sqlite3_str_appendf(&out, "%!.15g", pVar->u.r); +@@ -83600,6 +85146,26 @@ + # define UPDATE_MAX_BLOBSIZE(P) + #endif + ++#ifdef SQLITE_DEBUG ++/* This routine provides a convenient place to set a breakpoint during ++** tracing with PRAGMA vdbe_trace=on. The breakpoint fires right after ++** each opcode is printed. Variables "pc" (program counter) and pOp are ++** available to add conditionals to the breakpoint. GDB example: ++** ++** break test_trace_breakpoint if pc=22 ++** ++** Other useful labels for breakpoints include: ++** test_addop_breakpoint(pc,pOp) ++** sqlite3CorruptError(lineno) ++** sqlite3MisuseError(lineno) ++** sqlite3CantopenError(lineno) ++*/ ++static void test_trace_breakpoint(int pc, Op *pOp, Vdbe *v){ ++ static int n = 0; ++ n++; ++} ++#endif ++ + /* + ** Invoke the VDBE coverage callback, if that callback is defined. This + ** feature is used for test suite validation only and does not appear an +@@ -83678,14 +85244,6 @@ + } + #endif + +-/* +-** Convert the given register into a string if it isn't one +-** already. Return non-zero if a malloc() fails. +-*/ +-#define Stringify(P, enc) \ +- if(((P)->flags&(MEM_Str|MEM_Blob))==0 && sqlite3VdbeMemStringify(P,enc,0)) \ +- { goto no_mem; } +- + /* + ** An ephemeral string value (signified by the MEM_Ephem flag) contains + ** a pointer to a dynamically allocated string where some other entity +@@ -83747,7 +85305,7 @@ + ** is clear. Otherwise, if this is an ephemeral cursor created by + ** OP_OpenDup, the cursor will not be closed and will still be part + ** of a BtShared.pCursor list. */ +- p->apCsr[iCur]->isEphemeral = 0; ++ if( p->apCsr[iCur]->pBtx==0 ) p->apCsr[iCur]->isEphemeral = 0; + sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); + p->apCsr[iCur] = 0; + } +@@ -83767,6 +85325,21 @@ + return pCx; + } + ++/* ++** The string in pRec is known to look like an integer and to have a ++** floating point value of rValue. Return true and set *piValue to the ++** integer value if the string is in range to be an integer. Otherwise, ++** return false. ++*/ ++static int alsoAnInt(Mem *pRec, double rValue, i64 *piValue){ ++ i64 iValue = (double)rValue; ++ if( sqlite3RealSameAsInt(rValue,iValue) ){ ++ *piValue = iValue; ++ return 1; ++ } ++ return 0==sqlite3Atoi64(pRec->z, piValue, pRec->n, pRec->enc); ++} ++ + /* + ** Try to convert a value into a numeric representation if we can + ** do so without loss of information. In other words, if the string +@@ -83784,12 +85357,12 @@ + */ + static void applyNumericAffinity(Mem *pRec, int bTryForInt){ + double rValue; +- i64 iValue; + u8 enc = pRec->enc; +- assert( (pRec->flags & (MEM_Str|MEM_Int|MEM_Real))==MEM_Str ); +- if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return; +- if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){ +- pRec->u.i = iValue; ++ int rc; ++ assert( (pRec->flags & (MEM_Str|MEM_Int|MEM_Real|MEM_IntReal))==MEM_Str ); ++ rc = sqlite3AtoF(pRec->z, &rValue, pRec->n, enc); ++ if( rc<=0 ) return; ++ if( rc==1 && alsoAnInt(pRec, rValue, &pRec->u.i) ){ + pRec->flags |= MEM_Int; + }else{ + pRec->u.r = rValue; +@@ -83819,6 +85392,7 @@ + ** Convert pRec to a text representation. + ** + ** SQLITE_AFF_BLOB: ++** SQLITE_AFF_NONE: + ** No-op. pRec is unchanged. + */ + static void applyAffinity( +@@ -83843,11 +85417,14 @@ + ** there is already a string rep, but it is pointless to waste those + ** CPU cycles. */ + if( 0==(pRec->flags&MEM_Str) ){ /*OPTIMIZATION-IF-FALSE*/ +- if( (pRec->flags&(MEM_Real|MEM_Int)) ){ ++ if( (pRec->flags&(MEM_Real|MEM_Int|MEM_IntReal)) ){ ++ testcase( pRec->flags & MEM_Int ); ++ testcase( pRec->flags & MEM_Real ); ++ testcase( pRec->flags & MEM_IntReal ); + sqlite3VdbeMemStringify(pRec, enc, 1); + } + } +- pRec->flags &= ~(MEM_Real|MEM_Int); ++ pRec->flags &= ~(MEM_Real|MEM_Int|MEM_IntReal); + } + } + +@@ -83886,13 +85463,21 @@ + ** accordingly. + */ + static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ +- assert( (pMem->flags & (MEM_Int|MEM_Real))==0 ); ++ int rc; ++ sqlite3_int64 ix; ++ assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal))==0 ); + assert( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ); + ExpandBlob(pMem); +- if( sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc)==0 ){ +- return 0; +- } +- if( sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc)==0 ){ ++ rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); ++ if( rc<=0 ){ ++ if( rc==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1 ){ ++ pMem->u.i = ix; ++ return MEM_Int; ++ }else{ ++ return MEM_Real; ++ } ++ }else if( rc==1 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)==0 ){ ++ pMem->u.i = ix; + return MEM_Int; + } + return MEM_Real; +@@ -83906,10 +85491,15 @@ + ** But it does set pMem->u.r and pMem->u.i appropriately. + */ + static u16 numericType(Mem *pMem){ +- if( pMem->flags & (MEM_Int|MEM_Real) ){ +- return pMem->flags & (MEM_Int|MEM_Real); ++ if( pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal) ){ ++ testcase( pMem->flags & MEM_Int ); ++ testcase( pMem->flags & MEM_Real ); ++ testcase( pMem->flags & MEM_IntReal ); ++ return pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal); + } + if( pMem->flags & (MEM_Str|MEM_Blob) ){ ++ testcase( pMem->flags & MEM_Str ); ++ testcase( pMem->flags & MEM_Blob ); + return computeNumericType(pMem); + } + return 0; +@@ -83920,12 +85510,9 @@ + ** Write a nice string representation of the contents of cell pMem + ** into buffer zBuf, length nBuf. + */ +-SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){ +- char *zCsr = zBuf; ++SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, StrAccum *pStr){ + int f = pMem->flags; +- + static const char *const encnames[] = {"(X)", "(8)", "(16LE)", "(16BE)"}; +- + if( f&MEM_Blob ){ + int i; + char c; +@@ -83941,55 +85528,40 @@ + }else{ + c = 's'; + } +- *(zCsr++) = c; +- sqlite3_snprintf(100, zCsr, "%d[", pMem->n); +- zCsr += sqlite3Strlen30(zCsr); +- for(i=0; i<16 && i<pMem->n; i++){ +- sqlite3_snprintf(100, zCsr, "%02X", ((int)pMem->z[i] & 0xFF)); +- zCsr += sqlite3Strlen30(zCsr); ++ sqlite3_str_appendf(pStr, "%cx[", c); ++ for(i=0; i<25 && i<pMem->n; i++){ ++ sqlite3_str_appendf(pStr, "%02X", ((int)pMem->z[i] & 0xFF)); + } +- for(i=0; i<16 && i<pMem->n; i++){ ++ sqlite3_str_appendf(pStr, "|"); ++ for(i=0; i<25 && i<pMem->n; i++){ + char z = pMem->z[i]; +- if( z<32 || z>126 ) *zCsr++ = '.'; +- else *zCsr++ = z; ++ sqlite3_str_appendchar(pStr, 1, (z<32||z>126)?'.':z); + } +- *(zCsr++) = ']'; ++ sqlite3_str_appendf(pStr,"]"); + if( f & MEM_Zero ){ +- sqlite3_snprintf(100, zCsr,"+%dz",pMem->u.nZero); +- zCsr += sqlite3Strlen30(zCsr); ++ sqlite3_str_appendf(pStr, "+%dz",pMem->u.nZero); + } +- *zCsr = '\0'; + }else if( f & MEM_Str ){ +- int j, k; +- zBuf[0] = ' '; ++ int j; ++ u8 c; + if( f & MEM_Dyn ){ +- zBuf[1] = 'z'; ++ c = 'z'; + assert( (f & (MEM_Static|MEM_Ephem))==0 ); + }else if( f & MEM_Static ){ +- zBuf[1] = 't'; ++ c = 't'; + assert( (f & (MEM_Dyn|MEM_Ephem))==0 ); + }else if( f & MEM_Ephem ){ +- zBuf[1] = 'e'; ++ c = 'e'; + assert( (f & (MEM_Static|MEM_Dyn))==0 ); + }else{ +- zBuf[1] = 's'; ++ c = 's'; + } +- k = 2; +- sqlite3_snprintf(100, &zBuf[k], "%d", pMem->n); +- k += sqlite3Strlen30(&zBuf[k]); +- zBuf[k++] = '['; +- for(j=0; j<15 && j<pMem->n; j++){ +- u8 c = pMem->z[j]; +- if( c>=0x20 && c<0x7f ){ +- zBuf[k++] = c; +- }else{ +- zBuf[k++] = '.'; +- } ++ sqlite3_str_appendf(pStr, " %c%d[", c, pMem->n); ++ for(j=0; j<25 && j<pMem->n; j++){ ++ c = pMem->z[j]; ++ sqlite3_str_appendchar(pStr, 1, (c>=0x20&&c<=0x7f) ? c : '.'); + } +- zBuf[k++] = ']'; +- sqlite3_snprintf(100,&zBuf[k], encnames[pMem->enc]); +- k += sqlite3Strlen30(&zBuf[k]); +- zBuf[k++] = 0; ++ sqlite3_str_appendf(pStr, "]%s", encnames[pMem->enc]); + } + } + #endif +@@ -84005,29 +85577,48 @@ + printf(p->flags & MEM_Zero ? " NULL-nochng" : " NULL"); + }else if( (p->flags & (MEM_Int|MEM_Str))==(MEM_Int|MEM_Str) ){ + printf(" si:%lld", p->u.i); ++ }else if( (p->flags & (MEM_IntReal))!=0 ){ ++ printf(" ir:%lld", p->u.i); + }else if( p->flags & MEM_Int ){ + printf(" i:%lld", p->u.i); + #ifndef SQLITE_OMIT_FLOATING_POINT + }else if( p->flags & MEM_Real ){ +- printf(" r:%g", p->u.r); ++ printf(" r:%.17g", p->u.r); + #endif + }else if( sqlite3VdbeMemIsRowSet(p) ){ + printf(" (rowset)"); + }else{ +- char zBuf[200]; +- sqlite3VdbeMemPrettyPrint(p, zBuf); +- printf(" %s", zBuf); ++ StrAccum acc; ++ char zBuf[1000]; ++ sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); ++ sqlite3VdbeMemPrettyPrint(p, &acc); ++ printf(" %s", sqlite3StrAccumFinish(&acc)); + } + if( p->flags & MEM_Subtype ) printf(" subtype=0x%02x", p->eSubtype); + } + static void registerTrace(int iReg, Mem *p){ +- printf("REG[%d] = ", iReg); ++ printf("R[%d] = ", iReg); + memTracePrint(p); ++ if( p->pScopyFrom ){ ++ printf(" <== R[%d]", (int)(p->pScopyFrom - &p[-iReg])); ++ } + printf("\n"); + sqlite3VdbeCheckMemInvariants(p); + } + #endif + ++#ifdef SQLITE_DEBUG ++/* ++** Show the values of all registers in the virtual machine. Used for ++** interactive debugging. ++*/ ++SQLITE_PRIVATE void sqlite3VdbeRegisterDump(Vdbe *v){ ++ int i; ++ for(i=1; i<v->nMem; i++) registerTrace(i, v->aMem+i); ++} ++#endif /* SQLITE_DEBUG */ ++ ++ + #ifdef SQLITE_DEBUG + # define REGISTER_TRACE(R,M) if(db->flags&SQLITE_VdbeTrace)registerTrace(R,M) + #else +@@ -84056,7 +85647,7 @@ + ****************************************************************************** + ** + ** This file contains inline asm code for retrieving "high-performance" +-** counters for x86 class CPUs. ++** counters for x86 and x86_64 class CPUs. + */ + #ifndef SQLITE_HWTIME_H + #define SQLITE_HWTIME_H +@@ -84067,8 +85658,9 @@ + ** processor and returns that value. This can be used for high-res + ** profiling. + */ +-#if (defined(__GNUC__) || defined(_MSC_VER)) && \ +- (defined(i386) || defined(__i386__) || defined(_M_IX86)) ++#if !defined(__STRICT_ANSI__) && \ ++ (defined(__GNUC__) || defined(_MSC_VER)) && \ ++ (defined(i386) || defined(__i386__) || defined(_M_IX86)) + + #if defined(__GNUC__) + +@@ -84089,7 +85681,7 @@ + + #endif + +-#elif (defined(__GNUC__) && defined(__x86_64__)) ++#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long val; +@@ -84097,7 +85689,7 @@ + return val; + } + |