Hello!
We have Metabase instance connected to several DBs. One of them is Clickhouse, it has about hundered schemas and more than 1700 tables.
After last update (to the 0.51.3) we faced with several issues. Metabase is currently unable to synchronize new tables and their structure. I tried updating to 0.52.5, the problem still exists.
We mention that the sync-fields task has a status of "failed" with an error that looks something like this:
Task log details
{
"status": "failed",
"exception": "class clojure.lang.ExceptionInfo",
"message": "Code: 60. DB::Exception: Table <DB>.<TABLE> does not exist. Maybe you meant <DB>.<TABLE>?. (UNKNOWN_TABLE) (version 24.8.4.13 (official build))\n",
"stacktrace": [
"com.clickhouse.jdbc.SqlExceptionUtils.batchUpdateError(SqlExceptionUtils.java:107)",
"com.clickhouse.jdbc.internal.SqlBasedPreparedStatement.executeAny(SqlBasedPreparedStatement.java:219)",
"com.clickhouse.jdbc.internal.SqlBasedPreparedStatement.executeQuery(SqlBasedPreparedStatement.java:246)",
"com.mchange.v2.c3p0.impl.NewProxyPreparedStatement.executeQuery(NewProxyPreparedStatement.java:1471)",
"--> driver.sql_jdbc.sync.describe_table$fallback_fields_metadata_from_select_query$reify__89412.reduce(describe_table.clj:94)",
"driver.sql_jdbc.sync.describe_table$fields_metadata$reify$reify__89436.reduce(describe_table.clj:161)",
"driver.sql_jdbc.sync.describe_table$fields_metadata$reify__89431.reduce(describe_table.clj:169)",
"driver.sql_jdbc.sync.describe_table$fn__89452.invokeStatic(describe_table.clj:215)",
"driver.sql_jdbc.sync.describe_table$fn__89452.invoke(describe_table.clj:213)",
"driver.sql_jdbc.sync.describe_table$describe_table_STAR_.invokeStatic(describe_table.clj:283)",
"driver.sql_jdbc.sync.describe_table$describe_table_STAR_.invoke(describe_table.clj:279)",
"driver.sql_jdbc.sync.describe_table$describe_table$fn__89486.invoke(describe_table.clj:295)",
"driver.clickhouse$fn__118389$fn__118391.invoke(clickhouse.clj:126)",
"driver.sql_jdbc.execute$do_with_resolved_connection87060__87061.invokeStatic(execute.clj:338)",
"driver.sql_jdbc.execute$do_with_resolved_connection87060__87061.invoke(execute.clj:321)",
"driver.clickhouse$fn__118389.invokeStatic(clickhouse.clj:87)",
"driver.clickhouse$fn__118389.invoke(clickhouse.clj:85)",
"driver.sql_jdbc.sync.describe_table$describe_table.invokeStatic(describe_table.clj:290)",
"driver.sql_jdbc.sync.describe_table$describe_table.invoke(describe_table.clj:287)",
"driver.clickhouse_introspection$fn__114232.invokeStatic(clickhouse_introspection.clj:160)",
"driver.clickhouse_introspection$fn__114232.invoke(clickhouse_introspection.clj:158)",
"sync.fetch_metadata$table_fields_metadata90414__90415.invokeStatic(fetch_metadata.clj:53)",
"sync.fetch_metadata$table_fields_metadata90414__90415.invoke(fetch_metadata.clj:40)",
"sync.fetch_metadata$describe_fields_using_describe_table$fn__90421.invoke(fetch_metadata.clj:63)",
"db.connection$do_with_connection_primary_method_default.invokeStatic(connection.clj:132)",
"db.connection$do_with_connection_primary_method_default.invoke(connection.clj:130)",
"sync.sync_metadata.fields$sync_fields_BANG_90491__90492$fn__90493.invoke(fields.clj:82)",
"sync.util$do_with_error_handling.invokeStatic(util.clj:191)",
"sync.util$do_with_error_handling.invoke(util.clj:184)",
"sync.sync_metadata.fields$sync_fields_BANG_90491__90492.invokeStatic(fields.clj:76)",
"sync.sync_metadata.fields$sync_fields_BANG_90491__90492.invoke(fields.clj:71)",
"sync.util$run_step_with_metadata65871__65873$fn__65875$fn__65878.invoke(util.clj:495)",
"models.task_history$do_with_task_history65708__65709.invokeStatic(task_history.clj:121)",
"models.task_history$do_with_task_history65708__65709.invoke(task_history.clj:109)",
"sync.util$run_step_with_metadata65871__65873$fn__65875.doInvoke(util.clj:488)",
"sync.util$with_start_and_finish_logging_STAR_.invokeStatic(util.clj:132)",
"sync.util$with_start_and_finish_logging_STAR_.invoke(util.clj:126)",
"sync.util$do_with_start_and_finish_debug_logging.invokeStatic(util.clj:150)",
"sync.util$do_with_start_and_finish_debug_logging.invoke(util.clj:146)",
"sync.util$run_step_with_metadata65871__65873.invokeStatic(util.clj:482)",
"sync.util$run_step_with_metadata65871__65873.invoke(util.clj:477)",
"sync.util$run_sync_operation65921__65922$fn__65923$fn__65931.invoke(util.clj:568)",
"sync.util$run_sync_operation65921__65922$fn__65923.invoke(util.clj:566)",
"models.task_history$do_with_task_history65708__65709.invokeStatic(task_history.clj:121)",
"models.task_history$do_with_task_history65708__65709.invoke(task_history.clj:109)",
"sync.util$run_sync_operation65921__65922.invokeStatic(util.clj:563)",
"sync.util$run_sync_operation65921__65922.invoke(util.clj:558)",
"sync.sync_metadata$sync_db_metadata_BANG_91087__91088$fn__91089.invoke(sync_metadata.clj:70)",
"sync.util$do_with_error_handling.invokeStatic(util.clj:191)",
"sync.util$do_with_error_handling.invoke(util.clj:184)",
"driver$fn__58386.invokeStatic(driver.clj:892)",
"driver$fn__58386.invoke(driver.clj:892)",
"sync.util$sync_in_context$fn__65780.invoke(util.clj:167)",
"sync.util$with_db_logging_disabled$fn__65777.invoke(util.clj:159)",
"sync.util$with_start_and_finish_logging_STAR_.invokeStatic(util.clj:132)",
"sync.util$with_start_and_finish_logging_STAR_.invoke(util.clj:126)",
"sync.util$with_start_and_finish_logging$fn__65764.invoke(util.clj:144)",
"sync.util$with_sync_events65754__65755$fn__65759.invoke(util.clj:118)",
"sync.util$with_duplicate_ops_prevented$fn__65744.invoke(util.clj:90)",
"sync.util$do_sync_operation65795__65796.invokeStatic(util.clj:216)",
"sync.util$do_sync_operation65795__65796.invoke(util.clj:210)",
"sync.sync_metadata$sync_db_metadata_BANG_91087__91088.invokeStatic(sync_metadata.clj:68)",
"sync.sync_metadata$sync_db_metadata_BANG_91087__91088.invoke(sync_metadata.clj:65)",
"api.database$fn__104207$fn__104212.invoke(database.clj:1016)"
],
"ex-data": {
"toucan2/context-trace": [
[
"execute SQL with class com.mchange.v2.c3p0.impl.NewProxyConnection",
{
"toucan2.jdbc.query/sql-args": [
"SELECT * FROM `metabase_table` WHERE ((`active` = TRUE) AND (`visibility_type` IS NULL)) AND (`schema` IN (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)) AND (`db_id` = ?)",
"...",
14
]
}
],
[
"resolve connection",
{
"toucan2.connection/connectable": "class metabase.db.connection.ApplicationDB"
}
],
[
"resolve connection",
{
"toucan2.connection/connectable": "default"
}
],
[
"resolve connection",
{
"toucan2.connection/connectable": null
}
],
{
"toucan2.pipeline/rf": "clojure.core$map$fn__5950$fn__5951@47e3002d"
},
[
"with compiled query",
{
"toucan2.pipeline/compiled-query": [
"SELECT * FROM `metabase_table` WHERE ((`active` = TRUE) AND (`visibility_type` IS NULL)) AND (`schema` IN (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)) AND (`db_id` = ?)",
"...",
14
]
}
],
[
"with built query",
{
"toucan2.pipeline/built-query": {
"select": [
"*"
],
"from": [
[
"metabase_table"
]
],
"where": [
"and",
[
"and",
[
"=",
"active",
true
],
[
"=",
"visibility_type",
null
]
],
[
"in",
"schema",
[
"..."
]
],
null,
[
"=",
"db_id",
14
]
]
}
}
],
[
"with resolved query",
{
"toucan2.pipeline/resolved-query": {
"where": [
"and",
[
"and",
[
"=",
"active",
true
],
[
"=",
"visibility_type",
null
]
],
[
"in",
"schema",
[
"..."
]
],
null
]
}
}
],
[
"with parsed args",
{
"toucan2.pipeline/query-type": "toucan.query-type/select.instances",
"toucan2.pipeline/parsed-args": {
"kv-args": {
"db_id": 14
},
"queryable": {
"where": [
"and",
[
"and",
[
"=",
"active",
true
],
[
"=",
"visibility_type",
null
]
],
[
"in",
"schema",
[
"..."
]
],
null
]
}
}
}
],
[
"with model",
{
"toucan2.pipeline/model": "model/Table"
}
],
[
"with unparsed args",
{
"toucan2.pipeline/query-type": "toucan.query-type/select.instances",
"toucan2.pipeline/unparsed-args": [
"model/Table",
"db_id",
14,
{
"where": [
"and",
[
"and",
[
"=",
"active",
true
],
[
"=",
"visibility_type",
null
]
],
[
"in",
"schema",
[
"..."
]
],
null
]
}
]
}
]
]
},
"original-info": null
}
After several iterations of synchronizing the structure, some new tables did appear. But the "Table Metadata" page does not contain anything about their structure.
Our environment is very dynamic and the user can delete their tables at any time. Also, since it is a cluster, the tables may not be created correctly, which can lead to other types of errors.
We have not seen such problems before, although the environment has not changed.
Therefore, we believe that after such an error, the process should move on and this error message will simply be displayed in the log with the severity level WARN.
We tried to figure out where the problem is, but all the logs say nothing. Even after enabling the trace log, we can't find anything useful.
Can anyone help me understand what the root problem is and how to fix it?
Additional info about the instance
{
"browser-info": {
"language": "ru-RU",
"platform": "MacIntel",
"userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
"vendor": "Google Inc."
},
"metabase-info": {
"databases": [
"exasol",
"mysql",
"clickhouse",
"mongo"
],
"run-mode": "prod",
"plan-alias": "",
"version": {
"date": "2024-11-11",
"tag": "v0.51.3",
"hash": "d757d0b"
},
"settings": { },
"hosting-env": "unknown",
"application-database": "mysql",
"application-database-details": {
"database": {
"name": "MariaDB",
"version": "10.5.13-MariaDB-log"
},
"jdbc-driver": {
"name": "MariaDB Connector/J",
"version": "2.7.10"
}
}
},
"system-info": {
"file.encoding": "UTF-8",
"java.runtime.name": "OpenJDK Runtime Environment",
"java.runtime.version": "11.0.25+9",
"java.vendor": "Eclipse Adoptium",
"java.vendor.url": "https://adoptium.net/",
"java.version": "11.0.25",
"java.vm.name": "OpenJDK 64-Bit Server VM",
"java.vm.version": "11.0.25+9",
"os.name": "Linux",
"os.version": "5.14.0-497.el9.x86_64",
"user.language": "en",
}
}