Once or twice a day, there is a huge spike in CPU utilization for about a hour (from around 2-4% to 95-99%) and metabase crashes after that.
The database is a very small mongoDB with a about 200-400 documents.
Metabase is running on a t2.micro instance with 1GB ram and 1vCPU.
I could not find any pattern as to what causes a crash. Here are some instances of logs just before metabase crashes.
2020-11-17 05:48:22,481 INFO task.send-pulses :: Sending scheduled pulses...
2020-11-17 05:50:52,276 ERROR metabot.websocket :: Error connecting websocket:
clojure.lang.ExceptionInfo: slack.com: Temporary failure in name resolution
at metabase.integrations.slack$do_slack_request.invokeStatic(slack.clj:65) ~[metabase.jar:?]
at metabase.integrations.slack$do_slack_request.invoke(slack.clj:47) ~[metabase.jar:?]
at metabase.integrations.slack$GET.invokeStatic(slack.clj:70) ~[metabase.jar:?]
at metabase.integrations.slack$GET.doInvoke(slack.clj:67) ~[metabase.jar:?]
at clojure.lang.RestFn.invoke(RestFn.java:410) ~[metabase.jar:?]
at clojure.core$partial$fn__5839.invoke(core.clj:2623) ~[metabase.jar:?]
at clojure.core$comp$fn__5807.invoke(core.clj:2568) ~[metabase.jar:?]
at metabase.metabot.websocket$connect_websocket_BANG_.invokeStatic(websocket.clj:30) ~[metabase.jar:?]
at metabase.metabot.websocket$connect_websocket_BANG_.invoke(websocket.clj:29) ~[metabase.jar:?]
at metabase.metabot.websocket$reopen_websocket_connection_if_needed_BANG_.invokeStatic(websocket.clj:101) [metabase.jar:?]
at metabase.metabot.websocket$reopen_websocket_connection_if_needed_BANG_.invoke(websocket.clj:91) [metabase.jar:?]
at metabase.metabot.websocket$start_websocket_monitor_BANG_$fn__69913.invoke(websocket.clj:116) [metabase.jar:?]
at clojure.core$binding_conveyor_fn$fn__5754.invoke(core.clj:2030) [metabase.jar:?]
at clojure.lang.AFn.call(AFn.java:18) [metabase.jar:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
at java.lang.Thread.run(Thread.java:834) [?:?]
Caused by: java.net.UnknownHostException: slack.com: Temporary failure in name resolution
at java.net.Inet6AddressImpl.lookupAllHostAddr(Native Method) ~[?:?]
at java.net.InetAddress$PlatformNameService.lookupAllHostAddr(InetAddress.java:929) ~[?:?]
at java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1515) ~[?:?]
at java.net.InetAddress$NameServiceAddresses.get(InetAddress.java:848) ~[?:?]
at java.net.InetAddress.getAllByName0(InetAddress.java:1505) ~[?:?]
at java.net.InetAddress.getAllByName(InetAddress.java:1364) ~[?:?]
at java.net.InetAddress.getAllByName(InetAddress.java:1298) ~[?:?]
at org.apache.http.impl.conn.SystemDefaultDnsResolver.resolve(SystemDefaultDnsResolver.java:45) ~[metabase.jar:?]
at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:112) ~[metabase.jar:?]
at org.apache.http.impl.conn.BasicHttpClientConnectionManager.connect(BasicHttpClientConnectionManager.java:313) ~[metabase.jar:?]
at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:393) ~[metabase.jar:?]
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236) ~[metabase.jar:?]
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) ~[metabase.jar:?]
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) ~[metabase.jar:?]
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) ~[metabase.jar:?]
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) ~[metabase.jar:?]
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) ~[metabase.jar:?]
at clj_http.core$request.invokeStatic(core.clj:643) ~[metabase.jar:?]
at clj_http.core$request.invoke(core.clj:568) ~[metabase.jar:?]
at clj_http.core$request.invokeStatic(core.clj:569) ~[metabase.jar:?]
at clj_http.core$request.invoke(core.clj:568) ~[metabase.jar:?]
at clojure.lang.Var.invoke(Var.java:384) ~[metabase.jar:?]
at clj_http.client$wrap_request_timing$fn__23813.invoke(client.clj:1077) ~[metabase.jar:?]
at clj_http.headers$wrap_header_map$fn__22947.invoke(headers.clj:152) ~[metabase.jar:?]
at clj_http.client$wrap_query_params$fn__23703.invoke(client.clj:811) ~[metabase.jar:?]
at clj_http.client$wrap_basic_auth$fn__23709.invoke(client.clj:834) ~[metabase.jar:?]
at clj_http.client$wrap_oauth$fn__23714.invoke(client.clj:851) ~[metabase.jar:?]
at clj_http.client$wrap_user_info$fn__23723.invoke(client.clj:871) ~[metabase.jar:?]
at clj_http.client$wrap_url$fn__23795.invoke(client.clj:1029) ~[metabase.jar:?]
at clj_http.client$wrap_decompression$fn__23504.invoke(client.clj:418) ~[metabase.jar:?]
at clj_http.client$wrap_input_coercion$fn__23627.invoke(client.clj:631) ~[metabase.jar:?]
at clj_http.client$wrap_additional_header_parsing$fn__23652.invoke(client.clj:686) ~[metabase.jar:?]
at clj_http.client$wrap_output_coercion$fn__23614.invoke(client.clj:575) ~[metabase.jar:?]
at clj_http.client$wrap_exceptions$fn__23456.invoke(client.clj:252) ~[metabase.jar:?]
at clj_http.client$wrap_accept$fn__23667.invoke(client.clj:729) ~[metabase.jar:?]
at clj_http.client$wrap_accept_encoding$fn__23674.invoke(client.clj:751) ~[metabase.jar:?]
at clj_http.client$wrap_content_type$fn__23661.invoke(client.clj:712) ~[metabase.jar:?]
at clj_http.client$wrap_form_params$fn__23760.invoke(client.clj:953) ~[metabase.jar:?]
at clj_http.client$wrap_nested_params$fn__23781.invoke(client.clj:987) ~[metabase.jar:?]
at clj_http.client$wrap_flatten_nested_params$fn__23790.invoke(client.clj:1011) ~[metabase.jar:?]
at clj_http.client$wrap_method$fn__23728.invoke(client.clj:887) ~[metabase.jar:?]
at clj_http.cookies$wrap_cookies$fn__22859.invoke(cookies.clj:131) ~[metabase.jar:?]
at clj_http.links$wrap_links$fn__23283.invoke(links.clj:63) ~[metabase.jar:?]
at clj_http.client$wrap_unknown_host$fn__23798.invoke(client.clj:1040) ~[metabase.jar:?]
at clj_http.client$request_STAR_.invokeStatic(client.clj:1168) ~[metabase.jar:?]
at clj_http.client$request_STAR_.invoke(client.clj:1161) ~[metabase.jar:?]
at clj_http.client$get.invokeStatic(client.clj:1174) ~[metabase.jar:?]
at clj_http.client$get.doInvoke(client.clj:1170) ~[metabase.jar:?]
at clojure.lang.RestFn.invoke(RestFn.java:423) ~[metabase.jar:?]
at metabase.integrations.slack$do_slack_request.invokeStatic(slack.clj:63) ~[metabase.jar:?]
... 17 more
2020-11-15 09:11:26,807 DEBUG middleware.log :: e[32mPOST /api/dataset 202 [ASYNC: completed] 212.8 ms (23 DB calls) App DB connections: 0/15 Jetty threads: 2/50 (5 idle, 0 queued) (100 total active threads) Queries in flight: 0 (0 queued)e[0m
2020-11-15 09:11:36,511 ERROR middleware.log :: e[31mPOST /api/dataset 500 2.1 ms (0 DB calls)
{:via [{:type java.lang.Exception, :message "`database` is required for all queries whose type is not `internal`.", :at [metabase.api.dataset$fn__52570 invokeStatic "dataset.clj" 49]}],
:trace
[[metabase.api.dataset$fn__52570 invokeStatic "dataset.clj" 49]
[metabase.api.dataset$fn__52570 invoke "dataset.clj" 40]
[compojure.core$wrap_response$fn__1993 invoke "core.clj" 160]
[compojure.core$wrap_route_middleware$fn__1977 invoke "core.clj" 132]
[compojure.core$wrap_route_info$fn__1982 invoke "core.clj" 139]
[compojure.core$wrap_route_matches$fn__1986 invoke "core.clj" 151]
[compojure.core$routes$fn__2005$f__2006 invoke "core.clj" 198]
[compojure.core$routes$fn__2005$f__2006$respond_SINGLEQUOTE___2007 invoke "core.clj" 197]
[compojure.core$wrap_route_matches$fn__1986 invoke "core.clj" 152]
[compojure.core$routes$fn__2005$f__2006 invoke "core.clj" 198]
[compojure.core$routes$fn__2005 invoke "core.clj" 200]
[metabase.middleware.auth$enforce_authentication$fn__72857 invoke "auth.clj" 14]
[compojure.core$routes$fn__2005$f__2006 invoke "core.clj" 198]
[compojure.core$routes$fn__2005 invoke "core.clj" 200]
[compojure.core$make_context$handler__2033 invoke "core.clj" 287]
[compojure.core$make_context$fn__2035 invoke "core.clj" 296]
[compojure.core$routes$fn__2005$f__2006 invoke "core.clj" 198]
[compojure.core$routes$fn__2005$f__2006$respond_SINGLEQUOTE___2007 invoke "core.clj" 197]
[compojure.core$make_context$fn__2035 invoke "core.clj" 297]
[compojure.core$routes$fn__2005$f__2006 invoke "core.clj" 198]
[compojure.core$routes$fn__2005$f__2006$respond_SINGLEQUOTE___2007 invoke "core.clj" 197]
[compojure.core$make_context$fn__2035 invoke "core.clj" 297]
[compojure.core$routes$fn__2005$f__2006 invoke "core.clj" 198]
[compojure.core$routes$fn__2005$f__2006$respond_SINGLEQUOTE___2007 invoke "core.clj" 197]
[compojure.core$make_context$fn__2035 invoke "core.clj" 297]
[compojure.core$routes$fn__2005$f__2006 invoke "core.clj" 198]
[compojure.core$routes$fn__2005$f__2006$respond_SINGLEQUOTE___2007 invoke "core.clj" 197]
[compojure.core$make_context$fn__2035 invoke "core.clj" 297]
[compojure.core$routes$fn__2005$f__2006 invoke "core.clj" 198]
[compojure.core$routes$fn__2005$f__2006$respond_SINGLEQUOTE___2007 invoke "core.clj" 197]
[compojure.core$make_context$fn__2035 invoke "core.clj" 297]
[compojure.core$routes$fn__2005$f__2006 invoke "core.clj" 198]
[compojure.core$routes$fn__2005$f__2006$respond_SINGLEQUOTE___2007 invoke "core.clj" 197]
[compojure.core$make_context$fn__2035 invoke "core.clj" 297]
[compojure.core$routes$fn__2005$f__2006 invoke "core.clj" 198]
[compojure.core$routes$fn__2005$f__2006$respond_SINGLEQUOTE___2007 invoke "core.clj" 197]
[compojure.core$make_context$fn__2035 invoke "core.clj" 297]
[compojure.core$routes$fn__2005$f__2006 invoke "core.clj" 198]
[compojure.core$routes$fn__2005$f__2006$respond_SINGLEQUOTE___2007 invoke "core.clj" 197]
[metabase.api.routes$fn__72978$fn__72980 invoke "routes.clj" 73]
[compojure.core$routes$fn__2005$f__2006 invoke "core.clj" 198]
[compojure.core$routes$fn__2005 invoke "core.clj" 200]
[clojure.lang.AFn applyToHelper "AFn.java" 160]
[clojure.lang.AFn applyTo "AFn.java" 144]
[clojure.core$apply invokeStatic "core.clj" 665]
[clojure.core$apply invoke "core.clj" 660]
[metabase.routes$fn__74306$fn__74307 doInvoke "routes.clj" 60]
[clojure.lang.RestFn invoke "RestFn.java" 436]
[compojure.core$routes$fn__2005$f__2006 invoke "core.clj" 198]
[compojure.core$routes$fn__2005 invoke "core.clj" 200]
[compojure.core$make_context$handler__2033 invoke "core.clj" 287]
[compojure.core$make_context$fn__2035 invoke "core.clj" 296]
[compojure.core$routes$fn__2005$f__2006 invoke "core.clj" 198]
[compojure.core$routes$fn__2005$f__2006$respond_SINGLEQUOTE___2007 invoke "core.clj" 197]
[compojure.core$wrap_route_matches$fn__1986 invoke "core.clj" 153]
[compojure.core$routes$fn__2005$f__2006 invoke "core.clj" 198]
[compojure.core$routes$fn__2005$f__2006$respond_SINGLEQUOTE___2007 invoke "core.clj" 197]
[compojure.core$wrap_route_matches$fn__1986 invoke "core.clj" 153]
[compojure.core$routes$fn__2005$f__2006 invoke "core.clj" 198]
[compojure.core$routes$fn__2005$f__2006$respond_SINGLEQUOTE___2007 invoke "core.clj" 197]
[compojure.core$wrap_route_matches$fn__1986 invoke "core.clj" 153]
[compojure.core$routes$fn__2005$f__2006 invoke "core.clj" 198]
[compojure.core$routes$fn__2005$f__2006$respond_SINGLEQUOTE___2007 invoke "core.clj" 197]
[metabase.routes$fn__74294$fn__74296 invoke "routes.clj" 44]
[compojure.core$routes$fn__2005$f__2006 invoke "core.clj" 198]
[compojure.core$routes$fn__2005 invoke "core.clj" 200]
[metabase.middleware.exceptions$catch_uncaught_exceptions$fn__72969 invoke "exceptions.clj" 96]
[metabase.middleware.exceptions$catch_api_exceptions$fn__72966 invoke "exceptions.clj" 84]
[metabase.middleware.log$log_api_call$fn__74754$fn__74755 invoke "log.clj" 197]
[toucan.db$_do_with_call_counting invokeStatic "db.clj" 216]
[toucan.db$_do_with_call_counting invoke "db.clj" 209]
[metabase.middleware.log$log_api_call$fn__74754 invoke "log.clj" 191]
[metabase.middleware.security$add_security_headers$fn__72932 invoke "security.clj" 143]
[metabase.middleware.json$wrap_json_body$fn__74457 invoke "json.clj" 64]
[metabase.middleware.json$wrap_streamed_json_response$fn__74475 invoke "json.clj" 100]
[ring.middleware.keyword_params$wrap_keyword_params$fn__75020 invoke "keyword_params.clj" 55]
[ring.middleware.params$wrap_params$fn__75036 invoke "params.clj" 69]
[metabase.middleware.misc$maybe_set_site_url$fn__37860 invoke "misc.clj" 59]
[metabase.middleware.session$bind_current_user$fn__69411$fn__69412 invoke "session.clj" 278]
[metabase.middleware.session$do_with_current_user invokeStatic "session.clj" 260]
[metabase.middleware.session$do_with_current_user invoke "session.clj" 252]
[metabase.middleware.session$bind_current_user$fn__69411 invoke "session.clj" 277]
[metabase.middleware.session$wrap_current_user_info$fn__69398 invoke "session.clj" 238]
[metabase.middleware.session$wrap_session_id$fn__69384 invoke "session.clj" 184]
[metabase.middleware.auth$wrap_api_key$fn__72865 invoke "auth.clj" 27]
[ring.middleware.cookies$wrap_cookies$fn__74940 invoke "cookies.clj" 216]
[metabase.middleware.misc$add_content_type$fn__37845 invoke "misc.clj" 28]
[metabase.middleware.misc$disable_streaming_buffering$fn__37868 invoke "misc.clj" 76]
[ring.middleware.gzip$wrap_gzip$fn__74982 invoke "gzip.clj" 86]
[metabase.middleware.misc$bind_request$fn__37871 invoke "misc.clj" 93]
[metabase.middleware.ssl$redirect_to_https_middleware$fn__74771 invoke "ssl.clj" 39]
[metabase.server$async_proxy_handler$fn__74532 invoke "server.clj" 72]
[metabase.server.proxy$org.eclipse.jetty.server.handler.AbstractHandler$ff19274a handle nil -1]
[org.eclipse.jetty.server.handler.HandlerWrapper handle "HandlerWrapper.java" 127]
[org.eclipse.jetty.server.Server handle "Server.java" 516]
[org.eclipse.jetty.server.HttpChannel lambda$handle$1 "HttpChannel.java" 383]
[org.eclipse.jetty.server.HttpChannel dispatch "HttpChannel.java" 556]
[org.eclipse.jetty.server.HttpChannel handle "HttpChannel.java" 375]
[org.eclipse.jetty.server.HttpConnection onFillable "HttpConnection.java" 273]
[org.eclipse.jetty.io.AbstractConnection$ReadCallback succeeded "AbstractConnection.java" 311]
[org.eclipse.jetty.io.FillInterest fillable "FillInterest.java" 105]
[org.eclipse.jetty.io.ChannelEndPoint$1 run "ChannelEndPoint.java" 104]
[org.eclipse.jetty.util.thread.strategy.EatWhatYouKill runTask "EatWhatYouKill.java" 336]
[org.eclipse.jetty.util.thread.strategy.EatWhatYouKill doProduce "EatWhatYouKill.java" 313]
[org.eclipse.jetty.util.thread.strategy.EatWhatYouKill tryProduce "EatWhatYouKill.java" 171]
[org.eclipse.jetty.util.thread.strategy.EatWhatYouKill run "EatWhatYouKill.java" 129]
[org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread run "ReservedThreadExecutor.java" 375]
[org.eclipse.jetty.util.thread.QueuedThreadPool runJob "QueuedThreadPool.java" 773]
[org.eclipse.jetty.util.thread.QueuedThreadPool$Runner run "QueuedThreadPool.java" 905]
[java.lang.Thread run "Thread.java" 834]],
:cause "`database` is required for all queries whose type is not `internal`.",
:message "`database` is required for all queries whose type is not `internal`.",
:type java.lang.Exception}
e[0m
2020-11-15 09:11:38,737 DEBUG middleware.log :: e[32mPOST /api/dataset 202 [ASYNC: completed] 183.5 ms (42 DB calls) App DB connections: 0/15 Jetty threads: 2/50 (5 idle, 0 queued) (101 total active threads) Queries in flight: 0 (0 queued)e[0m
2020-11-15 09:11:57,887 ERROR middleware.catch-exceptions :: Error processing query: null
{:database_id 2,
:started_at #t "2020-11-15T09:11:57.601372+05:30[Asia/Kolkata]",
:error_type :unsupported-feature,
:json_query
{:type "query",
:query {:source-table 13, :aggregation [["count"]], :breakout [["fk->" ["field-id" 406] ["field-id" 48]]]},
:database 2,
:parameters [],
:middleware {:js-int-to-string? true, :add-default-userland-constraints? true}},
:native nil,
:status :failed,
:class clojure.lang.ExceptionInfo,
:stacktrace
["--> query_processor.middleware.add_implicit_joins$add_implicit_joins_STAR_.invokeStatic(add_implicit_joins.clj:240)"
"query_processor.middleware.add_implicit_joins$add_implicit_joins_STAR_.invoke(add_implicit_joins.clj:236)"
"query_processor.middleware.add_implicit_joins$add_implicit_joins$fn__40987.invoke(add_implicit_joins.clj:254)"
"query_processor.middleware.large_int_id$convert_id_to_string$fn__47385.invoke(large_int_id.clj:44)"
"query_processor.middleware.limit$limit$fn__47410.invoke(limit.clj:38)"
"query_processor.middleware.format_rows$format_rows$fn__47365.invoke(format_rows.clj:84)"
"query_processor.middleware.desugar$desugar$fn__46838.invoke(desugar.clj:22)"
"query_processor.middleware.binning$update_binning_strategy$fn__45859.invoke(binning.clj:229)"
"query_processor.middleware.resolve_fields$resolve_fields$fn__46376.invoke(resolve_fields.clj:24)"
"query_processor.middleware.add_dimension_projections$add_remapping$fn__40531.invoke(add_dimension_projections.clj:318)"
"query_processor.middleware.add_implicit_clauses$add_implicit_clauses$fn__40738.invoke(add_implicit_clauses.clj:141)"
"query_processor.middleware.add_source_metadata$add_source_metadata_for_source_queries$fn__41136.invoke(add_source_metadata.clj:105)"
"query_processor.middleware.reconcile_breakout_and_order_by_bucketing$reconcile_breakout_and_order_by_bucketing$fn__48304.invoke(reconcile_breakout_and_order_by_bucketing.clj:98)"
"query_processor.middleware.auto_bucket_datetimes$auto_bucket_datetimes$fn__45447.invoke(auto_bucket_datetimes.clj:125)"
"query_processor.middleware.resolve_source_table$resolve_source_tables$fn__46423.invoke(resolve_source_table.clj:46)"
"query_processor.middleware.parameters$substitute_parameters$fn__48089.invoke(parameters.clj:114)"
"query_processor.middleware.resolve_referenced$resolve_referenced_card_resources$fn__46475.invoke(resolve_referenced.clj:80)"
"query_processor.middleware.expand_macros$expand_macros$fn__47094.invoke(expand_macros.clj:158)"
"query_processor.middleware.add_timezone_info$add_timezone_info$fn__41167.invoke(add_timezone_info.clj:15)"
"query_processor.middleware.splice_params_in_response$splice_params_in_response$fn__49025.invoke(splice_params_in_response.clj:32)"
"query_processor.middleware.resolve_database_and_driver$resolve_database_and_driver$fn__48315$fn__48319.invoke(resolve_database_and_driver.clj:33)"
"driver$do_with_driver.invokeStatic(driver.clj:61)"
"driver$do_with_driver.invoke(driver.clj:57)"
"query_processor.middleware.resolve_database_and_driver$resolve_database_and_driver$fn__48315.invoke(resolve_database_and_driver.clj:27)"
"query_processor.middleware.fetch_source_query$resolve_card_id_source_tables$fn__47312.invoke(fetch_source_query.clj:267)"
"query_processor.middleware.store$initialize_store$fn__49034$fn__49035.invoke(store.clj:11)"
"query_processor.store$do_with_store.invokeStatic(store.clj:46)"
"query_processor.store$do_with_store.invoke(store.clj:40)"
"query_processor.middleware.store$initialize_store$fn__49034.invoke(store.clj:10)"
"query_processor.middleware.cache$maybe_return_cached_results$fn__46352.invoke(cache.clj:214)"
"query_processor.middleware.validate$validate_query$fn__49043.invoke(validate.clj:10)"
"query_processor.middleware.normalize_query$normalize$fn__47437.invoke(normalize_query.clj:22)"
"query_processor.middleware.add_rows_truncated$add_rows_truncated$fn__41005.invoke(add_rows_truncated.clj:36)"
"query_processor.middleware.results_metadata$record_and_return_metadata_BANG_$fn__49010.invoke(results_metadata.clj:147)"
"query_processor.middleware.constraints$add_default_userland_constraints$fn__46715.invoke(constraints.clj:42)"
"query_processor.middleware.process_userland_query$process_userland_query$fn__48178.invoke(process_userland_query.clj:136)"
"query_processor.middleware.catch_exceptions$catch_exceptions$fn__46658.invoke(catch_exceptions.clj:174)"
"query_processor.reducible$async_qp$qp_STAR___39794$thunk__39795.invoke(reducible.clj:103)"
"query_processor.reducible$async_qp$qp_STAR___39794.invoke(reducible.clj:109)"
"query_processor.reducible$sync_qp$qp_STAR___39803$fn__39806.invoke(reducible.clj:135)"
"query_processor.reducible$sync_qp$qp_STAR___39803.invoke(reducible.clj:134)"
"query_processor$process_userland_query.invokeStatic(query_processor.clj:237)"
"query_processor$process_userland_query.doInvoke(query_processor.clj:233)"
"query_processor$fn__49201$process_query_and_save_execution_BANG___49210$fn__49213.invoke(query_processor.clj:249)"
"query_processor$fn__49201$process_query_and_save_execution_BANG___49210.invoke(query_processor.clj:241)"
"query_processor$fn__49245$process_query_and_save_with_max_results_constraints_BANG___49254$fn__49257.invoke(query_processor.clj:261)"
"query_processor$fn__49245$process_query_and_save_with_max_results_constraints_BANG___49254.invoke(query_processor.clj:254)"
"api.dataset$fn__52570$fn__52573.invoke(dataset.clj:59)"
"query_processor.streaming$streaming_response_STAR_$fn__37098$fn__37099.invoke(streaming.clj:73)"
"query_processor.streaming$streaming_response_STAR_$fn__37098.invoke(streaming.clj:72)"
"async.streaming_response$do_f_STAR_.invokeStatic(streaming_response.clj:66)"
"async.streaming_response$do_f_STAR_.invoke(streaming_response.clj:64)"
"async.streaming_response$do_f_async$fn__24643.invoke(streaming_response.clj:85)"],
:context :ad-hoc,
:error ":mongo driver does not support foreign keys.",
:row_count 0,
:running_time 0,
:preprocessed nil,
:ex-data {:driver :mongo, :type :unsupported-feature},
:data {:rows [], :cols []}}
2020-11-15 09:11:57,894 DEBUG middleware.log :: e[32mPOST /api/dataset 202 [ASYNC: completed] 300.7 ms (6 DB calls) App DB connections: 0/15 Jetty threads: 2/50 (5 idle, 0 queued) (101 total active threads) Queries in flight: 1 (0 queued)e[0m
2020-11-15 09:12:12,616 DEBUG middleware.log :: e[32mPOST /api/dataset 202 [ASYNC: completed] 121.4 ms (12 DB calls) App DB connections: 0/15 Jetty threads: 2/50 (5 idle, 0 queued) (101 total active threads) Queries in flight: 0 (0 queued)e[0m
2020-11-15 09:12:26,153 DEBUG middleware.log :: e[32mPOST /api/dataset 202 [ASYNC: completed] 182.9 ms (42 DB calls) App DB connections: 0/15 Jetty threads: 2/50 (5 idle, 0 queued) (101 total active threads) Queries in flight: 0 (0 queued)e[0m
2020-11-15 09:13:21,663 DEBUG middleware.log :: e[32mPOST /api/dataset 202 [ASYNC: completed] 148.2 ms (31 DB calls) App DB connections: 0/15 Jetty threads: 2/50 (5 idle, 0 queued) (100 total active threads) Queries in flight: 1 (0 queued)e[0m
2020-11-15 10:00:00,002 INFO task.send-pulses :: Sending scheduled pulses...
2020-11-15 10:17:31,379 DEBUG middleware.log :: e[31mGET /api/user/current 401 194.5 µs (0 DB calls)
"Unauthenticated"
e[0m
2020-11-15 10:17:31,385 DEBUG middleware.log :: e[32mGET /api/session/properties 200 5.4 ms (0 DB calls) App DB connections: 0/15 Jetty threads: 4/50 (3 idle, 0 queued) (103 total active threads) Queries in flight: 0 (0 queued)e[0m
2020-11-15 10:51:30,658 WARN middleware.log :: e[31mPOST /api/jsonws/invoke 404 474.4 µs (0 DB calls)
"API endpoint does not exist."
e[0m
2020-11-15 11:00:00,007 INFO task.send-pulses :: Sending scheduled pulses...
2020-11-15 12:00:00,074 INFO task.send-pulses :: Sending scheduled pulses...
2020-11-15 12:00:00,067 ERROR core.JobRunShell :: Job DEFAULT.metabase.task.abandonment-emails.job threw an unhandled Exception:
clojure.lang.ExceptionInfo: Input to should-send-abandoment-email? does not match schema:
e[0;33m [nil (named (not (instance? java.time.temporal.Temporal a-metabase.models.user.UserInstance)) last-user) (named (not (instance? java.time.temporal.Temporal a-metabase.models.activity.ActivityInstance)) last-activity) (named (not (instance? java.time.temporal.Temporal a-metabase.models.view_log.ViewLogInstance)) last-view)] e[0m
at metabase.task.follow_up_emails$fn__78531$should_send_abandoment_email_QMARK___78540.invoke(follow_up_emails.clj:95) ~[metabase.jar:?]
at metabase.task.follow_up_emails$fn__78531$should_send_abandoment_email_QMARK___78540$fn__78541.invoke(follow_up_emails.clj:97) ~[metabase.jar:?]
at metabase.task.follow_up_emails$fn__78531$should_send_abandoment_email_QMARK___78540.invoke(follow_up_emails.clj:95) ~[metabase.jar:?]
at metabase.task.follow_up_emails$send_abandoment_email_if_needed_BANG_.invokeStatic(follow_up_emails.clj:119) ~[metabase.jar:?]
at metabase.task.follow_up_emails$send_abandoment_email_if_needed_BANG_.invoke(follow_up_emails.clj:114) ~[metabase.jar:?]
at metabase.task.follow_up_emails.AbandonmentEmail.execute(follow_up_emails.clj:135) ~[metabase.jar:?]
at org.quartz.core.JobRunShell.run(JobRunShell.java:213) [metabase.jar:?]
at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:557) [metabase.jar:?]
2020-11-15 12:00:00,148 ERROR core.ErrorLogger :: Job (DEFAULT.metabase.task.abandonment-emails.job threw an exception.
org.quartz.SchedulerException: Job threw an unhandled exception.
at org.quartz.core.JobRunShell.run(JobRunShell.java:224) [metabase.jar:?]
at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:557) [metabase.jar:?]
Caused by: clojure.lang.ExceptionInfo: Input to should-send-abandoment-email? does not match schema:
e[0;33m [nil (named (not (instance? java.time.temporal.Temporal a-metabase.models.user.UserInstance)) last-user) (named (not (instance? java.time.temporal.Temporal a-metabase.models.activity.ActivityInstance)) last-activity) (named (not (instance? java.time.temporal.Temporal a-metabase.models.view_log.ViewLogInstance)) last-view)] e[0m
at metabase.task.follow_up_emails$fn__78531$should_send_abandoment_email_QMARK___78540.invoke(follow_up_emails.clj:95) ~[metabase.jar:?]
at metabase.task.follow_up_emails$fn__78531$should_send_abandoment_email_QMARK___78540$fn__78541.invoke(follow_up_emails.clj:97) ~[metabase.jar:?]
at metabase.task.follow_up_emails$fn__78531$should_send_abandoment_email_QMARK___78540.invoke(follow_up_emails.clj:95) ~[metabase.jar:?]
at metabase.task.follow_up_emails$send_abandoment_email_if_needed_BANG_.invokeStatic(follow_up_emails.clj:119) ~[metabase.jar:?]
at metabase.task.follow_up_emails$send_abandoment_email_if_needed_BANG_.invoke(follow_up_emails.clj:114) ~[metabase.jar:?]
at metabase.task.follow_up_emails.AbandonmentEmail.execute(follow_up_emails.clj:135) ~[metabase.jar:?]
at org.quartz.core.JobRunShell.run(JobRunShell.java:213) ~[metabase.jar:?]
... 1 more
2020-11-15 12:27:21,337 DEBUG middleware.log :: e[32mGET /api/dashboard/1 200 14.7 ms (15 DB calls) App DB connections: 0/15 Jetty threads: 3/50 (4 idle, 0 queued) (100 total active threads) Queries in flight: 0 (0 queued)e[0m
2020-11-15 12:27:22,004 DEBUG middleware.log :: e[32mPOST /api/card/2/query 202 [ASYNC: completed] 518.0 ms (5 DB calls) App DB connections: 1/15 Jetty threads: 2/50 (5 idle, 0 queued) (129 total active threads) Queries in flight: 5 (0 queued)e[0m
2020-11-15 12:27:22,156 DEBUG middleware.log :: e[32mPOST /api/card/4/query 202 [ASYNC: completed] 494.4 ms (11 DB calls) App DB connections: 0/15 Jetty threads: 2/50 (5 idle, 0 queued) (126 total active threads) Queries in flight: 5 (0 queued)e[0m
2020-11-15 12:27:22,232 DEBUG middleware.log :: e[32mPOST /api/card/98/query 202 [ASYNC: completed] 573.3 ms (12 DB calls) App DB connections: 0/15 Jetty threads: 2/50 (5 idle, 0 queued) (130 total active threads) Queries in flight: 5 (0 queued)e[0m
2020-11-15 12:27:22,315 DEBUG middleware.log :: e[32mPOST /api/card/1/query 202 [ASYNC: completed] 658.0 ms (10 DB calls) App DB connections: 0/15 Jetty threads: 2/50 (5 idle, 0 queued) (128 total active threads) Queries in flight: 5 (0 queued)e[0m
2020-11-15 12:27:22,407 DEBUG middleware.log :: e[32mPOST /api/card/5/query 202 [ASYNC: completed] 748.2 ms (20 DB calls) App DB connections: 0/15 Jetty threads: 2/50 (5 idle, 0 queued) (127 total active threads) Queries in flight: 4 (0 queued)e[0m
2020-11-15 12:27:22,523 DEBUG middleware.log :: e[32mPOST /api/card/3/query 202 [ASYNC: completed] 884.4 ms (21 DB calls) App DB connections: 1/15 Jetty threads: 2/50 (5 idle, 0 queued) (127 total active threads) Queries in flight: 5 (0 queued)e[0m
2020-11-15 12:27:22,905 DEBUG middleware.log :: e[32mPOST /api/card/99/query 202 [ASYNC: completed] 781.9 ms (11 DB calls) App DB connections: 0/15 Jetty threads: 2/50 (5 idle, 0 queued) (131 total active threads) Queries in flight: 5 (0 queued)e[0m
2020-11-15 00:00:02,623 INFO sync.util :: e[35mFINISHED: Sync metadata for mongo Database 2 '****-prod' (2.6 s)e[0m
2020-11-13 13:31:08,218 DEBUG middleware.log :: e[32mPOST /api/card/99/query 202 [ASYNC: completed] 263.7 ms (11 DB calls) App DB connections: 0/13 Jetty threads: 2/50 (5 idle, 0 queued) (111 total active threads) Queries in flight: 0 (0 queued)e[0m
2020-11-13 14:00:00,002 INFO task.send-pulses :: Sending scheduled pulses...
Diagnostic info
{
"browser-info": {
"language": "en-US",
"platform": "Linux x86_64",
"userAgent": "Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:82.0) Gecko/20100101 Firefox/82.0",
"vendor": ""
},
"system-info": {
"file.encoding": "UTF-8",
"java.runtime.name": "OpenJDK Runtime Environment",
"java.runtime.version": "11.0.9.1+1-Ubuntu-0ubuntu1.20.04",
"java.vendor": "Ubuntu",
"java.vendor.url": "https://ubuntu.com/",
"java.version": "11.0.9.1",
"java.vm.name": "OpenJDK 64-Bit Server VM",
"java.vm.version": "11.0.9.1+1-Ubuntu-0ubuntu1.20.04",
"os.name": "Linux",
"os.version": "5.4.0-1029-aws",
"user.language": "en",
"user.timezone": "Asia/Kolkata"
},
"metabase-info": {
"databases": [
"mongo"
],
"hosting-env": "unknown",
"application-database": "h2",
"application-database-details": {
"database": {
"name": "H2",
"version": "1.4.197 (2018-03-18)"
},
"jdbc-driver": {
"name": "H2 JDBC Driver",
"version": "1.4.197 (2018-03-18)"
}
},
"run-mode": "prod",
"version": {
"date": "2020-10-26",
"tag": "v0.37.0.2",
"branch": "release-x.37.x",
"hash": "ba7be09"
},
"settings": {
"report-timezone": "Asia/Calcutta"
}
}
}
Any pointers on what could cause this and how we can fix this would be really appreciated. Thanks!