From 9ce320ef119d87c71ffaeed356530ecf00c8c1d3 Mon Sep 17 00:00:00 2001 From: luishou Date: Wed, 8 Jul 2026 16:25:37 +0800 Subject: [PATCH] =?UTF-8?q?docs:=20=E7=94=9F=E4=BA=A7=E5=BA=93=E6=97=A5?= =?UTF-8?q?=E6=9C=9F=E5=AD=97=E6=AE=B5=E8=BF=81=E7=A7=BB+=E6=B8=85?= =?UTF-8?q?=E6=B4=97=20SQL=20=E8=BF=90=E8=A1=8C=E6=89=8B=E5=86=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit customers.birthday/report_date 从脏 varchar 迁移为 date 类型, 原值备份到 *_raw 列,清理离谱年份。已在生产真实数据只读验证: birthday 解析 99.9%、report_date 98.9%。含备份/核对/回滚步骤。 Co-Authored-By: Claude Opus 4.8 --- sql/prod-date-migration.sql | 133 ++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 sql/prod-date-migration.sql diff --git a/sql/prod-date-migration.sql b/sql/prod-date-migration.sql new file mode 100644 index 0000000..2016094 --- /dev/null +++ b/sql/prod-date-migration.sql @@ -0,0 +1,133 @@ +-- ============================================================================= +-- 生产库 crm 日期字段迁移 + 清洗 +-- 目标:把 customers.birthday / report_date 从脏 varchar 迁移为 date 类型, +-- 原值备份到 *_raw 列,并清理离谱年份。全程非破坏(原值可回溯)。 +-- +-- 背景:测试库 crm_dev 已是 date 结构;生产库 crm 仍是 varchar 脏数据。 +-- 本脚本经生产真实数据只读验证:birthday 解析成功率 99.9%、report_date 98.9%, +-- 无法解析的原值保留在 *_raw 列,date 置 NULL。 +-- +-- ⚠️ 执行前必读: +-- 1. 【务必先整库备份】:mysqldump -h124.223.187.95 -ucrm -p crm > crm_backup_$(date +%F).sql +-- 2. 建议在业务低峰 / 维护窗口执行;应用可短暂停写。 +-- 3. 逐段执行,每段后看 STEP 7 的核对结果再继续。 +-- 4. MySQL 5.7 适用(未用 8.0 的 REGEXP_REPLACE)。 +-- ============================================================================= + +USE crm; + +-- ----------------------------------------------------------------------------- +-- STEP 0(可选):执行前预览将受影响的数据量(只读,不改数据) +-- ----------------------------------------------------------------------------- +SELECT + SUM(birthday IS NOT NULL AND TRIM(birthday)<>'') AS birthday_非空, + SUM(report_date IS NOT NULL AND TRIM(report_date)<>'') AS report_非空 +FROM customers; + + +-- ----------------------------------------------------------------------------- +-- STEP 1:新增原值备份列(幂等:若已存在请跳过本段) +-- ----------------------------------------------------------------------------- +ALTER TABLE customers + ADD COLUMN birthday_raw VARCHAR(255) NULL AFTER birthday, + ADD COLUMN report_date_raw VARCHAR(255) NULL AFTER report_date; + +-- 备份当前脏值 +UPDATE customers SET birthday_raw = birthday, report_date_raw = report_date; + + +-- ----------------------------------------------------------------------------- +-- STEP 2:规范化(从 *_raw 解析,写回 birthday/report_date 为 'YYYY-MM-DD' 或 NULL) +-- 规则:/ . → - ;合并多余 - ;去首尾 - ;8 位连写按 YYYYMMDD 解析; +-- 无法解析者置 NULL(原值仍在 *_raw)。 +-- ----------------------------------------------------------------------------- +UPDATE customers SET birthday = ( + CASE + WHEN birthday_raw IS NULL OR TRIM(birthday_raw) = '' THEN NULL + WHEN birthday_raw REGEXP '^[0-9]{8}$' + THEN DATE_FORMAT(STR_TO_DATE(birthday_raw, '%Y%m%d'), '%Y-%m-%d') + ELSE DATE_FORMAT( + STR_TO_DATE( + TRIM(BOTH '-' FROM + REPLACE(REPLACE(REPLACE(REPLACE(TRIM(birthday_raw),'/','-'),'.','-'),'--','-'),'--','-') + ), '%Y-%m-%d'), + '%Y-%m-%d') + END +); + +UPDATE customers SET report_date = ( + CASE + WHEN report_date_raw IS NULL OR TRIM(report_date_raw) = '' THEN NULL + WHEN report_date_raw REGEXP '^[0-9]{8}$' + THEN DATE_FORMAT(STR_TO_DATE(report_date_raw, '%Y%m%d'), '%Y-%m-%d') + ELSE DATE_FORMAT( + STR_TO_DATE( + TRIM(BOTH '-' FROM + REPLACE(REPLACE(REPLACE(REPLACE(TRIM(report_date_raw),'/','-'),'.','-'),'--','-'),'--','-') + ), '%Y-%m-%d'), + '%Y-%m-%d') + END +); + + +-- ----------------------------------------------------------------------------- +-- STEP 3:改列类型 varchar → date(此时列内均为 'YYYY-MM-DD' 或 NULL,转换干净) +-- ----------------------------------------------------------------------------- +ALTER TABLE customers MODIFY birthday DATE NULL; +ALTER TABLE customers MODIFY report_date DATE NULL; + + +-- ----------------------------------------------------------------------------- +-- STEP 4:清理离谱年份(仅动可见数据 deleted_at IS NULL;raw 已备份) +-- birthday 合理区间 [1930,2015](清掉 1905 那批 Excel 序列号垃圾等) +-- report_date 合理区间 [2010,2026](清掉 1955 / 5万年 等) +-- 如需连软删除行一并清理,去掉 deleted_at 条件即可。 +-- ----------------------------------------------------------------------------- +UPDATE customers SET birthday = NULL +WHERE deleted_at IS NULL AND birthday IS NOT NULL + AND (YEAR(birthday) < 1930 OR YEAR(birthday) > 2015); + +UPDATE customers SET report_date = NULL +WHERE deleted_at IS NULL AND report_date IS NOT NULL + AND (YEAR(report_date) < 2010 OR YEAR(report_date) > 2026); + + +-- ----------------------------------------------------------------------------- +-- STEP 5:人工抢救可明确还原的 birthday(按原值匹配,id 无关;仅当已置空时) +-- 其余含糊/垃圾值(5 位年、手机号、"美团加购"、1993-27 等)保持 NULL,不臆测。 +-- ----------------------------------------------------------------------------- +UPDATE customers SET birthday='1985-03-15' WHERE birthday IS NULL AND birthday_raw='1985-0315'; +UPDATE customers SET birthday='1987-05-08' WHERE birthday IS NULL AND birthday_raw='1987-5=8'; +UPDATE customers SET birthday='1984-01-13' WHERE birthday IS NULL AND birthday_raw='1984-·1-13'; +UPDATE customers SET birthday='1982-07-10' WHERE birthday IS NULL AND birthday_raw='1982-7-·10'; + + +-- ----------------------------------------------------------------------------- +-- STEP 6:迁移后核对(只读) +-- ----------------------------------------------------------------------------- +-- 6.1 类型应为 date +SHOW COLUMNS FROM customers WHERE Field IN ('birthday','report_date','birthday_raw','report_date_raw'); + +-- 6.2 数量与年份范围(离谱残留应为 0) +SELECT + SUM(birthday IS NOT NULL) AS birthday_有值, + MIN(YEAR(birthday)) AS 生日最小年, + MAX(YEAR(birthday)) AS 生日最大年, + SUM(report_date IS NOT NULL) AS 报单_有值, + MIN(YEAR(report_date)) AS 报单最小年, + MAX(YEAR(report_date)) AS 报单最大年 +FROM customers WHERE deleted_at IS NULL; + +-- 6.3 未能解析、置空但原值仍在的清单(供人工复核,date 空 + raw 有值) +SELECT id, birthday_raw FROM customers WHERE deleted_at IS NULL AND birthday IS NULL AND TRIM(birthday_raw) <> ''; +SELECT id, report_date_raw FROM customers WHERE deleted_at IS NULL AND report_date IS NULL AND TRIM(report_date_raw) <> ''; + + +-- ============================================================================= +-- 回滚方案(如迁移后发现问题) +-- A. 值回滚(列类型已改为 date 时):把原始脏值写回不可行(date 存不下脏串), +-- 应从 STEP 0 前的 mysqldump 整表恢复: +-- mysql -h... -ucrm -p crm < crm_backup_YYYY-MM-DD.sql +-- B. 仅想恢复某几行的原始文本:查 *_raw 列即为迁移前原值。 +-- ⇒ 因此 STEP 0 的整库备份是回滚的唯一可靠依据,务必先做。 +-- =============================================================================