fix(db): introspect the Oracle schema in a fixed number of queries [10.16]

.github/workflows/php-unit.yml

@@ -88,6 +88,28 @@ jobs:
         run: |
           make install-composer-deps
 
+      - name: TEMPORARY DEBUG - Oracle client
+        if: startsWith(matrix.database,'oracle:')
+        env:
+          DSN: "(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=127.0.0.1)(PORT=1521))(CONNECT_DATA=(SERVICE_NAME=FREEPDB1)))"
+        run: |
+          php -r 'echo "oci8: ", phpversion("oci8"), "\n";'
+          php -i | grep -iE "oracle|instant" || true
+          ls -l /opt/hostedtoolcache/setup-php/*/instantclient* 2>/dev/null || true
+          ldd "$(php -r 'echo ini_get("extension_dir");')/oci8.so" || true
+          echo "--- raw tcp to 1521 ---"
+          timeout 10 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/1521' && echo "tcp ok" || echo "tcp failed"
+          echo "--- oci_connect (60s budget) ---"
+          timeout -s KILL 60 php -d display_errors=1 -r '
+            $t = microtime(true);
+            $c = @oci_connect("owncloud", "owncloud", getenv("DSN"));
+            printf("connect=%s after %.1fs\n", $c ? "OK" : "FAIL", microtime(true) - $t);
+            if (!$c) { print_r(oci_error()); exit(1); }
+            $s = oci_parse($c, "select banner from v\$version");
+            oci_execute($s);
+            print_r(oci_fetch_row($s));
+          ' || echo "EXIT=$? (124/137 means it hung)"
+
       - name: Install Server
         env:
           DATA_DIRECTORY: ${{ github.workspace }}/data
@@ -126,7 +148,63 @@ jobs:
               --database-pass=${DB_PASSWORD}"
           fi
 
-          php occ ${install_cmd}
+          if [[ "${DB_TYPE}" == "oci" ]]; then
+            export DSN="(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=${DB_HOST})(PORT=1521))(CONNECT_DATA=(SERVICE_NAME=${ORACLE_SERVICE_NAME})))"
+            cat > /tmp/oracle-watch.php <<'WATCHEOF'
+          <?php
+          $c = @oci_connect('system', 'owncloud', getenv('DSN'));
+          if (!$c) { echo "  (no system connection)\n"; exit; }
+          $q = function ($c, $sql) {
+            $s = oci_parse($c, $sql);
+            oci_execute($s);
+            return oci_fetch_row($s)[0];
+          };
+          printf("  objects=%s introspections=%s dict_secs=%s\n",
+            $q($c, "select count(*) from dba_objects where owner = 'OWNCLOUD'"),
+            $q($c, "select nvl(sum(executions), 0) from v\$sql"
+              . " where parsing_schema_name = 'OWNCLOUD' and sql_text like 'SELECT * FROM sys.user_tables%'"),
+            $q($c, "select round(nvl(sum(elapsed_time), 0) / 1e6) from v\$sql"
+              . " where parsing_schema_name = 'OWNCLOUD'"));
+          $s = oci_parse($c, "select s.sid, s.status, s.state, s.event, s.seconds_in_wait,"
+            . " substr(q.sql_text, 1, 90) txt"
+            . " from v\$session s left join v\$sql q on q.sql_id = s.sql_id"
+            . " where s.username = 'OWNCLOUD'");
+          oci_execute($s);
+          while ($r = oci_fetch_assoc($s)) {
+            printf("    sid=%s %s/%s wait=%ss event=%s sql=%s\n",
+              $r['SID'], $r['STATUS'], $r['STATE'], $r['SECONDS_IN_WAIT'],
+              $r['EVENT'], preg_replace('/\s+/', ' ', (string)$r['TXT']));
+          }
+          WATCHEOF
+            cat > /tmp/oracle-top-sql.php <<'TOPEOF'
+          <?php
+          $c = @oci_connect('system', 'owncloud', getenv('DSN'));
+          if (!$c) { echo "  (no system connection)\n"; exit; }
+          $s = oci_parse($c, "select executions, round(elapsed_time/1e6, 1) secs,"
+            . " round(cpu_time/1e6, 1) cpu, substr(sql_text, 1, 110) txt"
+            . " from v\$sql where parsing_schema_name = 'OWNCLOUD'"
+            . " order by elapsed_time desc fetch first 12 rows only");
+          oci_execute($s);
+          echo "  --- top SQL by elapsed time (schema OWNCLOUD) ---\n";
+          while ($r = oci_fetch_assoc($s)) {
+            printf("  execs=%-6s elapsed=%-8ss cpu=%-8ss %s\n",
+              $r['EXECUTIONS'], $r['SECS'], $r['CPU'], preg_replace('/\s+/', ' ', $r['TXT']));
+          }
+          TOPEOF
+            timeout -s KILL 5400 php occ ${install_cmd} &
+            INSTALL_PID=$!
+            START=$(date +%s)
+            while kill -0 $INSTALL_PID 2>/dev/null; do
+              sleep 60
+              echo "  --- elapsed=$(( $(date +%s) - START ))s ---"
+              timeout -s KILL 30 php /tmp/oracle-watch.php || true
+            done
+            wait $INSTALL_PID && INSTALL_RC=0 || INSTALL_RC=$?
+            timeout -s KILL 60 php /tmp/oracle-top-sql.php || true
+            if [[ $INSTALL_RC -ne 0 ]]; then exit $INSTALL_RC; fi
+          else
+            php occ ${install_cmd}
+          fi
           echo "enabling apps"
           php occ app:enable files_sharing
           php occ app:enable files_trashbin

changelog/unreleased/41815-oracle-path-hash

@@ -0,0 +1,13 @@
+Bugfix: Avoid a deprecation notice when hashing the file cache path on Oracle
+
+Oracle cannot store empty strings, so the file cache converts them to null
+before writing a row. For the storage root, whose path is the empty string, that
+left md5() being called with null. PHP 8 reports that as a deprecated implicit
+null to string conversion: noise in the log whenever a storage root is inserted,
+and an error under PHPUnit's strict error handling. The stored path_hash itself
+was never wrong, because md5(null) coerces to md5('').
+
+The value is now cast to a string before hashing.
+
+https://github.com/owncloud/core/pull/41808
+https://github.com/owncloud/core/pull/41815

changelog/unreleased/41819

@@ -0,0 +1,30 @@
+Bugfix: Speed up Oracle schema introspection
+
+Installing and upgrading ownCloud on Oracle took an unreasonably long time. A
+fresh `occ maintenance:install` on the 10.16 branch needed over 40 minutes,
+while the same install on the master branch finished in well under a minute.
+
+The cause was the bundled doctrine/dbal 2.13, which introspects a schema by
+describing every table on its own: for each table it issues one query for the
+columns, one for the indexes, one for the foreign keys and one for the table
+comment. Each of those queries inlines the table name as a literal, so Oracle
+cannot share cursors between them and hard parses every single one, which
+costs a few hundred milliseconds each. The migration code then asks for the
+full schema once per applied migration, so the number of queries grows with
+the number of tables multiplied by the number of migrations. With 68
+migrations and roughly 48 tables that added up to thousands of hard parsed
+queries.
+
+Oracle schema introspection now reads the whole data dictionary with a fixed
+number of queries instead of four per table. Reading a 48 table schema went
+down from 194 queries to 6, and `occ maintenance:install` against Oracle on
+PHP 7.4 went down from 43 minutes to 30 seconds. The resulting schema is
+unchanged; it is compared against the previous implementation in the test
+suite.
+
+doctrine/dbal does the same thing natively from version 3.4 onwards, which is
+why the master branch was never affected. Upgrading doctrine/dbal on the 10.16
+branch is not an option, because its 3.x line changes public API that
+third-party apps use.
+
+https://github.com/owncloud/core/pull/41819

composer.json

@@ -2,6 +2,10 @@
     "name": "owncloud/core",
     "description": "A safe home for all your data",
     "license": "AGPL-3.0-or-later",
+    "repositories": [
+        { "type": "vcs", "url": "https://github.com/DeepDiver1975/streams" },
+        { "type": "vcs", "url": "https://github.com/DeepDiver1975/SMB" }
+    ],
     "config" : {
         "vendor-dir": "lib/composer",
         "optimize-autoloader": true,

lib/private/DB/Connection.php

@@ -35,6 +35,7 @@
 use Doctrine\Common\EventManager;
 use Doctrine\DBAL\Driver\ServerInfoAwareConnection;
 use Doctrine\DBAL\Platforms\MySqlPlatform;
+use Doctrine\DBAL\Platforms\OraclePlatform;
 use Doctrine\DBAL\Schema\Schema;
 use OC\DB\QueryBuilder\QueryBuilder;
 use OCP\DB\QueryBuilder\IQueryBuilder;
@@ -63,6 +64,20 @@ public function connect() {
 		}
 	}
 
+	/**
+	 * On Oracle, use a schema manager that introspects the whole schema with a
+	 * constant number of queries instead of four per table. See
+	 * {@see \OC\DB\OracleSchemaManager} for why this is needed.
+	 *
+	 * @return \Doctrine\DBAL\Schema\AbstractSchemaManager
+	 */
+	public function getSchemaManager() {
+		if ($this->_schemaManager === null && $this->getDatabasePlatform() instanceof OraclePlatform) {
+			$this->_schemaManager = new OracleSchemaManager($this, $this->getDatabasePlatform());
+		}
+		return parent::getSchemaManager();
+	}
+
 	/**
 	 * Returns a QueryBuilder for the connection.
 	 *