Merge pull request #31 from rustprooflabs/dev

Partitions, database view, etc.

Author: rustprooflabs

.cargo/config

@@ -1,16 +1,22 @@
 # Auto-generated by pgx. You may edit this, or delete it to have a new one created.
 
 [target.x86_64-unknown-linux-gnu]
-linker = "./.cargo/pgx-linker-script.sh"
+linker = ".cargo/pgx-linker-script.sh"
 
 [target.aarch64-unknown-linux-gnu]
-linker = "./.cargo/pgx-linker-script.sh"
+linker = ".cargo/pgx-linker-script.sh"
+
+[target.x86_64-unknown-linux-musl]
+linker = ".cargo/pgx-linker-script.sh"
+
+[target.aarch64-unknown-linux-musl]
+linker = ".cargo/pgx-linker-script.sh"
 
 [target.x86_64-apple-darwin]
-linker = "./.cargo/pgx-linker-script.sh"
+linker = ".cargo/pgx-linker-script.sh"
 
 [target.aarch64-apple-darwin]
-linker = "./.cargo/pgx-linker-script.sh"
+linker = ".cargo/pgx-linker-script.sh"
 
 [target.x86_64-unknown-freebsd]
-linker = "./.cargo/pgx-linker-script.sh"
+linker = ".cargo/pgx-linker-script.sh"

ADVANCED-INSTALL.md

@@ -74,33 +74,34 @@ changes the last two digits of the port!
 
 
 ```bash
-cargo pgx run pg13
+cargo pgx run pg14
 ```
 
-Example output.
+The output starts with something similar to:
 
 ```bash
-    Stopping Postgres v13
-building extension with features `pg13`
-"cargo" "build" "--features" "pg13" "--no-default-features"
-    Finished dev [unoptimized + debuginfo] target(s) in 0.05s
+building extension with features `pg14`
+"cargo" "build" "--lib" "--features" "pg14" "--no-default-features"
+    Updating crates.io index
+```
+
 
-installing extension
-     Copying control file to `/home/username/.pgx/13.4/pgx-install/share/postgresql/extension/pgdd.control`
-     Copying shared library to `/home/username/.pgx/13.4/pgx-install/lib/postgresql/pgdd.so`
-    Building SQL generator with features `pg13`
-"cargo" "build" "--bin" "sql-generator" "--features" "pg13" "--no-default-features"
-    Finished dev [unoptimized + debuginfo] target(s) in 0.05s
+```bash
+     Copying control file to `/home/username/.pgx/14.0/pgx-install/share/postgresql/extension/pgdd.control`
+     Copying shared library to `/home/username/.pgx/14.0/pgx-install/lib/postgresql/pgdd.so`
+    Building SQL generator with features `pg14`
+"cargo" "build" "--bin" "sql-generator" "--features" "pg14" "--no-default-features"
+   Compiling pgdd v0.4.0 (/home/username/git/pgdd)
+    Finished dev [unoptimized + debuginfo] target(s) in 13.84s
  Discovering SQL entities
   Discovered 9 SQL entities: 0 schemas (0 unique), 6 functions, 0 types, 0 enums, 3 sqls, 0 ords, 0 hashes
-running SQL generator with features `pg13`
-"cargo" "run" "--bin" "sql-generator" "--features" "pg13" "--no-default-features" "--" "--sql" "/home/username/.pgx/13.4/pgx-install/share/postgresql/extension/pgdd--0.4.0-dev.sql"
+running SQL generator with features `pg14`
+"cargo" "run" "--bin" "sql-generator" "--features" "pg14" "--no-default-features" "--" "--sql" "/home/username/.pgx/14.0/pgx-install/share/postgresql/extension/pgdd--0.4.1-dev.sql"
     Finished dev [unoptimized + debuginfo] target(s) in 0.06s
-     Running `target/debug/sql-generator --sql /home/username/.pgx/13.4/pgx-install/share/postgresql/extension/pgdd--0.4.0-dev.sql`
-     Copying extension schema file to `/home/username/.pgx/13.4/pgx-install/share/postgresql/extension/pgdd--0.3.1--0.4.0-dev.sql`
-     Copying extension schema file to `/home/username/.pgx/13.4/pgx-install/share/postgresql/extension/pgdd--0.3--0.3.1.sql`
+     Running `target/debug/sql-generator --sql /home/username/.pgx/14.0/pgx-install/share/postgresql/extension/pgdd--0.4.1-dev.sql`
+     Copying extension schema file to `/home/username/.pgx/14.0/pgx-install/share/postgresql/extension/pgdd--0.4.1-dev.sql`
     Finished installing pgdd
-    Starting Postgres v13 on port 28813
+    Starting Postgres v14 on port 28814
     Re-using existing database pgdd
 ```
 
@@ -110,6 +111,8 @@ In the test instance of psql, create the extension in database.
 CREATE EXTENSION pgdd;
 ```
 
+> Note: When you see "Re-using existing database pgdd" your previous installed version of `pgdd` will be available. To ensure you are working with the latest version of the `pgdd` extension you must drop/create the extension, quit the psql shell, and re-run the `cargo pgx run` command.
+
 
 ## Build binary packages
 
@@ -135,10 +138,10 @@ cp ./target/artifacts/* ./standalone/
 
 ```bash
 cargo pgx schema -d
-dot -Goverlap=prism -Gspline=ortho -Tjpg extension.dot > extension.jpg
+dot -Goverlap=prism -Gspline=ortho -Tjpg extension.dot > pgdd.jpg
 ```
 
-![pgx dependencies for pgdd v0.4.0](pgdd--0.4.0.jpg)
+![pgx dependencies for pgdd](pgdd.jpg)
 
 
 ## Non-standard dev

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pgdd"
-version = "0.4.0"
+version = "0.4.1-dev"
 edition = "2018"
 description = "In-database (PostgreSQL) data dictionary providing database introspection via standard SQL query syntax."
 
@@ -20,15 +20,15 @@ pg_test = []
 
 #pgx = { git = "https://github.com/zombodb/pgx", branch = "develop" }
 #pgx-macros = { git = "https://github.com/zombodb/pgx", branch = "develop" }
-pgx = "0.2.5"
-pgx-macros = "0.2.5"
+pgx = "0.3.3"
+pgx-macros = "0.3.3"
 
 #pgx-utils = { git = "https://github.com/zombodb/pgx", branch = "develop" }
-pgx-utils = "0.2.5"
+pgx-utils = "0.3.3"
 
 [dev-dependencies]
 #pgx-tests = { git = "https://github.com/zombodb/pgx", branch = "develop" }
-pgx-tests = "0.2.5"
+pgx-tests = "0.3.3"
 
 
 [profile.dev]

README.md

@@ -12,41 +12,6 @@ The extension is built on the Rust [pgx framework](https://github.com/zombodb/pg
 PgDD has been tested to work for PostgreSQL 10 through 14.
 
 
-## Upgrading from <= v0.3
-
-Version 0.4.0 was a complete rewrite of the PgDD extension from a raw-SQL
-extension to using the [pgx framework](https://github.com/zombodb/pgx). 
-
-
-
-Upgrading from Raw SQL version of PgDD (v0.3) to the pgx version (v0.4.0) is done with `DROP EXTENSION pgdd; CREATE EXTENSION pgdd;`
-Care has been taken to provide a smooth upgrade experience but
-**do not install the upgrade without testing the process on a test server!**
-
-If custom attributes were stored in the `dd` tables you will need to use
-`pg_dump` to export the data and reload after recreating the extension
-with pgx.  If any of the three (3) queries below return a count > 0
-this applies to you.
-
-
-```sql
-SELECT COUNT(*)
-    FROM dd.meta_table
-    WHERE s_name <> 'dd';
-SELECT COUNT(*)
-    FROM dd.meta_column
-    WHERE s_name <> 'dd';
-SELECT COUNT(*)
-    FROM dd.meta_schema
-    WHERE s_name <> 'dd';
-```
-
-
-
-The last raw SQL version is still available to [download](https://raw.githubusercontent.com/rustprooflabs/pgdd/main/standalone/pgdd_v0_3.sql).  This version is no longer maintained and may or may not
-work on future Postgres versions.
-
-
 ## Install from binary
 
 Binaries are available for Ubuntu 20.04 (focal) and Ubuntu 21.04 (hirsute).
@@ -209,6 +174,55 @@ SELECT s_name, f_name, argument_data_types, result_data_types
 ;
 ```
 
+### Partitioned tables
+
+There are two views, ``dd.partition_parents`` and ``dd.partition_children`` to provide
+partition-focused details.  Will display partitions for both
+declarative partitions and inheritance based partitions
+
+
+
+With the test data in this project for declarative partitions.
+
+
+```sql
+SELECT *
+    FROM dd.partition_parents
+    WHERE s_name = 'pgdd_test'
+;
+```
+
+```
+┌───────┬───────────┬────────┬────────────────┬────────────┬────────────┬─────────────┬────────────────────┬──────┬────────────────────┬───────────────────────────┬────────────────────┐
+│  oid  │  s_name   │ t_name │ partition_type │ partitions │ size_bytes │ size_pretty │ size_per_partition │ rows │ rows_per_partition │ partitions_never_analyzed │ partitions_no_data │
+╞═══════╪═══════════╪════════╪════════════════╪════════════╪════════════╪═════════════╪════════════════════╪══════╪════════════════════╪═══════════════════════════╪════════════════════╡
+│ 25090 │ pgdd_test │ parent │ declarative    │          3 │      40960 │ 40 kB       │ 13 kB              │   15 │                  5 │                         0 │                  1 │
+└───────┴───────────┴────────┴────────────────┴────────────┴────────────┴─────────────┴────────────────────┴──────┴────────────────────┴───────────────────────────┴────────────────────┘
+```
+
+Details for each child partition, including calculated percentages of the single
+partition against the totals for the parent partition.
+
+
+
+```sql
+SELECT *
+    FROM dd.partition_children
+    WHERE s_name = 'pgdd_test'
+;
+```
+
+```
+┌───────┬───────────┬─────────────┬────────────┬──────────────────┬──────┬────────────┬─────────────┬───────────────────┬───────────────┬───────────────────────────┬────────────────────────────┐
+│  oid  │  s_name   │   t_name    │ parent_oid │   parent_name    │ rows │ size_bytes │ size_pretty │ size_plus_indexes │ bytes_per_row │ percent_of_partition_rows │ percent_of_partition_bytes │
+╞═══════╪═══════════╪═════════════╪════════════╪══════════════════╪══════╪════════════╪═════════════╪═══════════════════╪═══════════════╪═══════════════════════════╪════════════════════════════╡
+│ 25095 │ pgdd_test │ child_0_10  │      25090 │ pgdd_test.parent │    9 │      16384 │ 16 kB       │ 32 kB             │          1820 │                    0.6000 │                     0.4000 │
+│ 25109 │ pgdd_test │ child_20_30 │      25090 │ pgdd_test.parent │    0 │       8192 │ 8192 bytes  │ 16 kB             │             ¤ │                    0.0000 │                     0.2000 │
+│ 25102 │ pgdd_test │ child_10_20 │      25090 │ pgdd_test.parent │    6 │      16384 │ 16 kB       │ 32 kB             │          2731 │                    0.4000 │                     0.4000 │
+└───────┴───────────┴─────────────┴────────────┴──────────────────┴──────┴────────────┴─────────────┴───────────────────┴───────────────┴───────────────────────────┴────────────────────────────┘
+```
+
+
 
 
 
@@ -253,6 +267,46 @@ GRANT dd_readwrite TO <your_login_user>;
 
 
 
+## Upgrade extension
+
+Version 0.4.0 was a complete rewrite of the PgDD extension from a raw-SQL
+extension to using the [pgx framework](https://github.com/zombodb/pgx).
+
+
+Upgrading versions currently requires `DROP EXTENSION pgdd; CREATE EXTENSION pgdd;`
+to recreate the extension.
+This is unlikely to change until [pgx #121 is resolved](https://github.com/zombodb/pgx/issues/121).
+
+
+
+If custom attributes were stored in the `dd` tables you will need to use
+`pg_dump` to export the data and reload after recreating the extension
+with pgx.  If any of the three (3) queries below return a count > 0
+this applies to you.
+
+
+```sql
+SELECT COUNT(*)
+    FROM dd.meta_table
+    WHERE s_name <> 'dd';
+SELECT COUNT(*)
+    FROM dd.meta_column
+    WHERE s_name <> 'dd';
+SELECT COUNT(*)
+    FROM dd.meta_schema
+    WHERE s_name <> 'dd';
+```
+
+
+
+The last raw SQL version (v0.3) is available to [download](https://raw.githubusercontent.com/rustprooflabs/pgdd/main/standalone/pgdd_v0_3.sql).  This version is no longer maintained and may or may not
+work on future Postgres versions.
+
+
+
+----
+
+
 ## PgDD UI
 
 The [PgDD UI](https://github.com/rustprooflabs/pgdd-ui) project provides
@@ -265,6 +319,8 @@ PgDD versioning must conform to
 [PEP 440](https://www.python.org/dev/peps/pep-0440/).
 
 
+----
+
 ## Caveats
 
 End user caveats:
@@ -273,6 +329,10 @@ End user caveats:
 
 Extension developer caveats:
 
-* DDL changes made in `src/lib.rs` need to be in version-to-version upgrade (e.g. ``sql/pgdd-0.3.1--0.4.0.sql``)
+* DDL changes made in `src/lib.rs` need to be in version-to-version upgrade (e.g. ``sql/pgdd-0.3.1--0.4.0.sql``).  This trick is not being used due to
+[pgx #121](https://github.com/zombodb/pgx/issues/121). The Pgx functionality is exposed
+via functions, and the functions are the core problem here.
+
+
 
 

build/build.sh

@@ -21,7 +21,7 @@ BASE=$(dirname `pwd`)
 VERSION=$(cat $BASE/pgdd.control | grep default_version | cut -f2 -d\')
 LOGDIR=${BASE}/target/logs
 ARTIFACTDIR=${BASE}/target/artifacts
-PGXVERSION=0.2.5
+PGXVERSION=0.3.3
 
 PG_VERS=("pg10" "pg11" "pg12" "pg13" "pg14")
 #PG_VERS=("pg14")

…ld/docker/pgdd-ubuntu-hirsute/Dockerfile …ild/docker/pgdd-ubuntu-impish/Dockerfilebuild/docker/pgdd-ubuntu-hirsute/Dockerfile renamed to build/docker/pgdd-ubuntu-impish/Dockerfile build/docker/pgdd-ubuntu-hirsute/Dockerfile renamed to build/docker/pgdd-ubuntu-impish/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:hirsute
+FROM ubuntu:impish
 
 LABEL maintainer="PgDD Project - https://github.com/rustprooflabs/pgdd"
 
@@ -12,22 +12,23 @@ RUN useradd -m ${USER} --uid=${UID}
 
 ARG DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get upgrade -y && apt-get install -y make wget curl gnupg git
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main" >> /etc/apt/sources.list.d/pgdg.list \
-    && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-RUN apt-get update && apt-get upgrade -y
-RUN apt-get install -y \
-        clang-12 llvm-12 clang libz-dev strace pkg-config \
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ impish-pgdg main" >> /etc/apt/sources.list.d/pgdg.list \
+    &&  curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null
+RUN apt-get update && apt-get upgrade -y --fix-missing
+RUN apt-get install -y --fix-missing \
+        clang-13 llvm-13 clang libz-dev strace pkg-config \
         libxml2 libxml2-dev libreadline8 libreadline-dev \
         flex bison libbison-dev build-essential \
         zlib1g-dev libxslt-dev libssl-dev libxml2-utils xsltproc libgss-dev \
-        libldap2-dev libkrb5-dev gettext tcl-tclreadline tcl-dev libperl-dev \
+        libldap-dev libkrb5-dev gettext tcl-tclreadline tcl-dev libperl-dev \
         libpython3-dev libprotobuf-c-dev libprotobuf-dev gcc \
         ruby ruby-dev rubygems \
         postgresql-server-dev-10 \
         postgresql-server-dev-11 \
         postgresql-server-dev-12 \
         postgresql-server-dev-13 \
-        postgresql-server-dev-14
+        postgresql-server-dev-14 \
+    && apt autoremove -y
 
 
 RUN gem install --no-document fpm
@@ -41,10 +42,6 @@ ENV PATH="/home/${USER}/.cargo/bin:${PATH}"
 
 RUN /bin/bash rustup.sh -y \
     && cargo install cargo-pgx --version ${PGXVERSION}
-#RUN /bin/bash rustup.sh -y \
-#    && cargo install --force --git "https://github.com/zombodb/pgx" \
-#        --branch "develop" \
-#        "cargo-pgx"
 
 
 RUN mkdir -p /home/${USER}/.pgx/data-10 \

pgdd--0.4.0.jpg

@@ -1,5 +1,5 @@
 comment = 'An in-database data dictionary providing database introspection via standard SQL query syntax.  Developed using pgx (https://github.com/zombodb/pgx).'
-default_version = '0.4.0'
+default_version = '0.4.1-dev'
 module_pathname = '$libdir/pgdd'
 relocatable = false
 schema = dd