From d00a55772cd1762e295673a3d8f5ee0b58f9b964 Mon Sep 17 00:00:00 2001 From: Bharathy Date: Sat, 17 Sep 2022 07:39:04 +0530 Subject: [PATCH] error compressing wide table Consider a compressed hypertable has many columns (like more than 600 columns). In call to compress_chunk(), the compressed tuple size exceeds, 8K which causes error as "row is too big: size 10856, maximum size 8160." This patch estimates the tuple size of compressed hypertable and reports a warning when compression is enabled on hypertable. Thus user gets aware of this warning before calling compress_chunk(). Fixes #4398 --- CHANGELOG.md | 1 + src/hypertable.c | 23 ++++++++++++++++++++++- tsl/test/expected/compression_errors.out | 21 +++++++++++++++++++++ tsl/test/sql/compression_errors.sql | 15 ++++++++++++++- 4 files changed, 58 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dbdf9dae..0a401e755 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ argument or resolve the type ambiguity by casting to the intended type. * #4673 Fix now() constification for VIEWs * #4681 Fix compression_chunk_size primary key * #4685 Improve chunk exclusion for space dimensions +* #4696 Report warning when enabling compression on hypertable **Thanks** * @maxtwardowski for reporting problems with chunk exclusion and space dimensions diff --git a/src/hypertable.c b/src/hypertable.c index 04f7a0135..72fad723b 100644 --- a/src/hypertable.c +++ b/src/hypertable.c @@ -2549,8 +2549,29 @@ ts_hypertable_create_compressed(Oid table_relid, int32 hypertable_id) NameData schema_name, table_name, associated_schema_name; ChunkSizingInfo *chunk_sizing_info; Relation rel; - rel = table_open(table_relid, AccessExclusiveLock); + int32 row_size = MAXALIGN(SizeofHeapTupleHeader); + /* estimate tuple width of compressed hypertable */ + for (int i = 1; i <= RelationGetNumberOfAttributes(rel); i++) + { + bool is_varlena = false; + Oid outfunc; + Form_pg_attribute att = TupleDescAttr(rel->rd_att, i - 1); + getTypeOutputInfo(att->atttypid, &outfunc, &is_varlena); + if (is_varlena) + row_size += 18; + else + row_size += att->attlen; + } + if (row_size > MaxHeapTupleSize) + { + ereport(WARNING, + (errmsg("compressed row size might exceed maximum row size"), + errdetail("Estimated row size of compressed hypertable is %u. This exceeds the " + "maximum size of %zu and can cause compression of chunks to fail.", + row_size, + MaxHeapTupleSize))); + } /* * Check that the user has permissions to make this table to a compressed * hypertable diff --git a/tsl/test/expected/compression_errors.out b/tsl/test/expected/compression_errors.out index 04860b812..03bb96a2f 100644 --- a/tsl/test/expected/compression_errors.out +++ b/tsl/test/expected/compression_errors.out @@ -611,3 +611,24 @@ EXPLAIN SELECT DISTINCT 1 FROM test; -> Seq Scan on compress_hyper_30_20_chunk (cost=0.00..1.02 rows=2 width=4) (4 rows) +--github issue 4398 +SELECT format('CREATE TABLE data_table AS SELECT now() AS tm, %s', array_to_string(array_agg(format('125 AS c%s',a)), ', ')) FROM generate_series(1,550)a \gexec +CREATE TABLE data_table AS SELECT now() AS tm, 125 AS c1, 125 AS c2, 125 AS c3, 125 AS c4, 125 AS c5, 125 AS c6, 125 AS c7, 125 AS c8, 125 AS c9, 125 AS c10, 125 AS c11, 125 AS c12, 125 AS c13, 125 AS c14, 125 AS c15, 125 AS c16, 125 AS c17, 125 AS c18, 125 AS c19, 125 AS c20, 125 AS c21, 125 AS c22, 125 AS c23, 125 AS c24, 125 AS c25, 125 AS c26, 125 AS c27, 125 AS c28, 125 AS c29, 125 AS c30, 125 AS c31, 125 AS c32, 125 AS c33, 125 AS c34, 125 AS c35, 125 AS c36, 125 AS c37, 125 AS c38, 125 AS c39, 125 AS c40, 125 AS c41, 125 AS c42, 125 AS c43, 125 AS c44, 125 AS c45, 125 AS c46, 125 AS c47, 125 AS c48, 125 AS c49, 125 AS c50, 125 AS c51, 125 AS c52, 125 AS c53, 125 AS c54, 125 AS c55, 125 AS c56, 125 AS c57, 125 AS c58, 125 AS c59, 125 AS c60, 125 AS c61, 125 AS c62, 125 AS c63, 125 AS c64, 125 AS c65, 125 AS c66, 125 AS c67, 125 AS c68, 125 AS c69, 125 AS c70, 125 AS c71, 125 AS c72, 125 AS c73, 125 AS c74, 125 AS c75, 125 AS c76, 125 AS c77, 125 AS c78, 125 AS c79, 125 AS c80, 125 AS c81, 125 AS c82, 125 AS c83, 125 AS c84, 125 AS c85, 125 AS c86, 125 AS c87, 125 AS c88, 125 AS c89, 125 AS c90, 125 AS c91, 125 AS c92, 125 AS c93, 125 AS c94, 125 AS c95, 125 AS c96, 125 AS c97, 125 AS c98, 125 AS c99, 125 AS c100, 125 AS c101, 125 AS c102, 125 AS c103, 125 AS c104, 125 AS c105, 125 AS c106, 125 AS c107, 125 AS c108, 125 AS c109, 125 AS c110, 125 AS c111, 125 AS c112, 125 AS c113, 125 AS c114, 125 AS c115, 125 AS c116, 125 AS c117, 125 AS c118, 125 AS c119, 125 AS c120, 125 AS c121, 125 AS c122, 125 AS c123, 125 AS c124, 125 AS c125, 125 AS c126, 125 AS c127, 125 AS c128, 125 AS c129, 125 AS c130, 125 AS c131, 125 AS c132, 125 AS c133, 125 AS c134, 125 AS c135, 125 AS c136, 125 AS c137, 125 AS c138, 125 AS c139, 125 AS c140, 125 AS c141, 125 AS c142, 125 AS c143, 125 AS c144, 125 AS c145, 125 AS c146, 125 AS c147, 125 AS c148, 125 AS c149, 125 AS c150, 125 AS c151, 125 AS c152, 125 AS c153, 125 AS c154, 125 AS c155, 125 AS c156, 125 AS c157, 125 AS c158, 125 AS c159, 125 AS c160, 125 AS c161, 125 AS c162, 125 AS c163, 125 AS c164, 125 AS c165, 125 AS c166, 125 AS c167, 125 AS c168, 125 AS c169, 125 AS c170, 125 AS c171, 125 AS c172, 125 AS c173, 125 AS c174, 125 AS c175, 125 AS c176, 125 AS c177, 125 AS c178, 125 AS c179, 125 AS c180, 125 AS c181, 125 AS c182, 125 AS c183, 125 AS c184, 125 AS c185, 125 AS c186, 125 AS c187, 125 AS c188, 125 AS c189, 125 AS c190, 125 AS c191, 125 AS c192, 125 AS c193, 125 AS c194, 125 AS c195, 125 AS c196, 125 AS c197, 125 AS c198, 125 AS c199, 125 AS c200, 125 AS c201, 125 AS c202, 125 AS c203, 125 AS c204, 125 AS c205, 125 AS c206, 125 AS c207, 125 AS c208, 125 AS c209, 125 AS c210, 125 AS c211, 125 AS c212, 125 AS c213, 125 AS c214, 125 AS c215, 125 AS c216, 125 AS c217, 125 AS c218, 125 AS c219, 125 AS c220, 125 AS c221, 125 AS c222, 125 AS c223, 125 AS c224, 125 AS c225, 125 AS c226, 125 AS c227, 125 AS c228, 125 AS c229, 125 AS c230, 125 AS c231, 125 AS c232, 125 AS c233, 125 AS c234, 125 AS c235, 125 AS c236, 125 AS c237, 125 AS c238, 125 AS c239, 125 AS c240, 125 AS c241, 125 AS c242, 125 AS c243, 125 AS c244, 125 AS c245, 125 AS c246, 125 AS c247, 125 AS c248, 125 AS c249, 125 AS c250, 125 AS c251, 125 AS c252, 125 AS c253, 125 AS c254, 125 AS c255, 125 AS c256, 125 AS c257, 125 AS c258, 125 AS c259, 125 AS c260, 125 AS c261, 125 AS c262, 125 AS c263, 125 AS c264, 125 AS c265, 125 AS c266, 125 AS c267, 125 AS c268, 125 AS c269, 125 AS c270, 125 AS c271, 125 AS c272, 125 AS c273, 125 AS c274, 125 AS c275, 125 AS c276, 125 AS c277, 125 AS c278, 125 AS c279, 125 AS c280, 125 AS c281, 125 AS c282, 125 AS c283, 125 AS c284, 125 AS c285, 125 AS c286, 125 AS c287, 125 AS c288, 125 AS c289, 125 AS c290, 125 AS c291, 125 AS c292, 125 AS c293, 125 AS c294, 125 AS c295, 125 AS c296, 125 AS c297, 125 AS c298, 125 AS c299, 125 AS c300, 125 AS c301, 125 AS c302, 125 AS c303, 125 AS c304, 125 AS c305, 125 AS c306, 125 AS c307, 125 AS c308, 125 AS c309, 125 AS c310, 125 AS c311, 125 AS c312, 125 AS c313, 125 AS c314, 125 AS c315, 125 AS c316, 125 AS c317, 125 AS c318, 125 AS c319, 125 AS c320, 125 AS c321, 125 AS c322, 125 AS c323, 125 AS c324, 125 AS c325, 125 AS c326, 125 AS c327, 125 AS c328, 125 AS c329, 125 AS c330, 125 AS c331, 125 AS c332, 125 AS c333, 125 AS c334, 125 AS c335, 125 AS c336, 125 AS c337, 125 AS c338, 125 AS c339, 125 AS c340, 125 AS c341, 125 AS c342, 125 AS c343, 125 AS c344, 125 AS c345, 125 AS c346, 125 AS c347, 125 AS c348, 125 AS c349, 125 AS c350, 125 AS c351, 125 AS c352, 125 AS c353, 125 AS c354, 125 AS c355, 125 AS c356, 125 AS c357, 125 AS c358, 125 AS c359, 125 AS c360, 125 AS c361, 125 AS c362, 125 AS c363, 125 AS c364, 125 AS c365, 125 AS c366, 125 AS c367, 125 AS c368, 125 AS c369, 125 AS c370, 125 AS c371, 125 AS c372, 125 AS c373, 125 AS c374, 125 AS c375, 125 AS c376, 125 AS c377, 125 AS c378, 125 AS c379, 125 AS c380, 125 AS c381, 125 AS c382, 125 AS c383, 125 AS c384, 125 AS c385, 125 AS c386, 125 AS c387, 125 AS c388, 125 AS c389, 125 AS c390, 125 AS c391, 125 AS c392, 125 AS c393, 125 AS c394, 125 AS c395, 125 AS c396, 125 AS c397, 125 AS c398, 125 AS c399, 125 AS c400, 125 AS c401, 125 AS c402, 125 AS c403, 125 AS c404, 125 AS c405, 125 AS c406, 125 AS c407, 125 AS c408, 125 AS c409, 125 AS c410, 125 AS c411, 125 AS c412, 125 AS c413, 125 AS c414, 125 AS c415, 125 AS c416, 125 AS c417, 125 AS c418, 125 AS c419, 125 AS c420, 125 AS c421, 125 AS c422, 125 AS c423, 125 AS c424, 125 AS c425, 125 AS c426, 125 AS c427, 125 AS c428, 125 AS c429, 125 AS c430, 125 AS c431, 125 AS c432, 125 AS c433, 125 AS c434, 125 AS c435, 125 AS c436, 125 AS c437, 125 AS c438, 125 AS c439, 125 AS c440, 125 AS c441, 125 AS c442, 125 AS c443, 125 AS c444, 125 AS c445, 125 AS c446, 125 AS c447, 125 AS c448, 125 AS c449, 125 AS c450, 125 AS c451, 125 AS c452, 125 AS c453, 125 AS c454, 125 AS c455, 125 AS c456, 125 AS c457, 125 AS c458, 125 AS c459, 125 AS c460, 125 AS c461, 125 AS c462, 125 AS c463, 125 AS c464, 125 AS c465, 125 AS c466, 125 AS c467, 125 AS c468, 125 AS c469, 125 AS c470, 125 AS c471, 125 AS c472, 125 AS c473, 125 AS c474, 125 AS c475, 125 AS c476, 125 AS c477, 125 AS c478, 125 AS c479, 125 AS c480, 125 AS c481, 125 AS c482, 125 AS c483, 125 AS c484, 125 AS c485, 125 AS c486, 125 AS c487, 125 AS c488, 125 AS c489, 125 AS c490, 125 AS c491, 125 AS c492, 125 AS c493, 125 AS c494, 125 AS c495, 125 AS c496, 125 AS c497, 125 AS c498, 125 AS c499, 125 AS c500, 125 AS c501, 125 AS c502, 125 AS c503, 125 AS c504, 125 AS c505, 125 AS c506, 125 AS c507, 125 AS c508, 125 AS c509, 125 AS c510, 125 AS c511, 125 AS c512, 125 AS c513, 125 AS c514, 125 AS c515, 125 AS c516, 125 AS c517, 125 AS c518, 125 AS c519, 125 AS c520, 125 AS c521, 125 AS c522, 125 AS c523, 125 AS c524, 125 AS c525, 125 AS c526, 125 AS c527, 125 AS c528, 125 AS c529, 125 AS c530, 125 AS c531, 125 AS c532, 125 AS c533, 125 AS c534, 125 AS c535, 125 AS c536, 125 AS c537, 125 AS c538, 125 AS c539, 125 AS c540, 125 AS c541, 125 AS c542, 125 AS c543, 125 AS c544, 125 AS c545, 125 AS c546, 125 AS c547, 125 AS c548, 125 AS c549, 125 AS c550 +CREATE TABLE ts_table (LIKE data_table); +SELECT * FROM create_hypertable('ts_table', 'tm'); +NOTICE: adding not-null constraint to column "tm" +DETAIL: Time dimensions cannot have NULL values. + hypertable_id | schema_name | table_name | created +---------------+-------------+------------+--------- + 31 | public | ts_table | t +(1 row) + +--should report a warning +\set VERBOSITY terse +ALTER TABLE ts_table SET(timescaledb.compress, timescaledb.compress_segmentby = 'c1', + timescaledb.compress_orderby = 'tm'); +WARNING: compressed row size might exceed maximum row size +INSERT INTO ts_table SELECT * FROM data_table; +--cleanup tables +DROP TABLE data_table cascade; +DROP TABLE ts_table cascade; diff --git a/tsl/test/sql/compression_errors.sql b/tsl/test/sql/compression_errors.sql index 20651b987..3ce8469d1 100644 --- a/tsl/test/sql/compression_errors.sql +++ b/tsl/test/sql/compression_errors.sql @@ -347,4 +347,17 @@ SELECT COMPRESS_CHUNK(X) FROM SHOW_CHUNKS('test') X; --below query should pass after chunks are compressed SELECT 1 FROM test GROUP BY enum_col; -EXPLAIN SELECT DISTINCT 1 FROM test; \ No newline at end of file +EXPLAIN SELECT DISTINCT 1 FROM test; + +--github issue 4398 +SELECT format('CREATE TABLE data_table AS SELECT now() AS tm, %s', array_to_string(array_agg(format('125 AS c%s',a)), ', ')) FROM generate_series(1,550)a \gexec +CREATE TABLE ts_table (LIKE data_table); +SELECT * FROM create_hypertable('ts_table', 'tm'); +--should report a warning +\set VERBOSITY terse +ALTER TABLE ts_table SET(timescaledb.compress, timescaledb.compress_segmentby = 'c1', + timescaledb.compress_orderby = 'tm'); +INSERT INTO ts_table SELECT * FROM data_table; +--cleanup tables +DROP TABLE data_table cascade; +DROP TABLE ts_table cascade;