Files
liandong-crm/sql/prod-date-migration.sql
T
luishouandClaude Opus 4.8 9ce320ef11 docs: 生产库日期字段迁移+清洗 SQL 运行手册
customers.birthday/report_date 从脏 varchar 迁移为 date 类型,
原值备份到 *_raw 列,清理离谱年份。已在生产真实数据只读验证:
birthday 解析 99.9%、report_date 98.9%。含备份/核对/回滚步骤。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-08 16:25:37 +08:00

134 lines
6.9 KiB
SQL
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
-- =============================================================================
-- 生产库 crm 日期字段迁移 + 清洗
-- 目标:把 customers.birthday / report_date 从脏 varchar 迁移为 date 类型,
-- 原值备份到 *_raw 列,并清理离谱年份。全程非破坏(原值可回溯)。
--
-- 背景:测试库 crm_dev 已是 date 结构;生产库 crm 仍是 varchar 脏数据。
-- 本脚本经生产真实数据只读验证:birthday 解析成功率 99.9%、report_date 98.9%,
-- 无法解析的原值保留在 *_raw 列,date 置 NULL。
--
-- ⚠️ 执行前必读:
-- 1. 【务必先整库备份】:mysqldump -h124.223.187.95 -ucrm -p crm > crm_backup_$(date +%F).sql
-- 2. 建议在业务低峰 / 维护窗口执行;应用可短暂停写。
-- 3. 逐段执行,每段后看 STEP 7 的核对结果再继续。
-- 4. MySQL 5.7 适用(未用 8.0 的 REGEXP_REPLACE)。
-- =============================================================================
USE crm;
-- -----------------------------------------------------------------------------
-- STEP 0(可选):执行前预览将受影响的数据量(只读,不改数据)
-- -----------------------------------------------------------------------------
SELECT
SUM(birthday IS NOT NULL AND TRIM(birthday)<>'') AS birthday_非空,
SUM(report_date IS NOT NULL AND TRIM(report_date)<>'') AS report_非空
FROM customers;
-- -----------------------------------------------------------------------------
-- STEP 1:新增原值备份列(幂等:若已存在请跳过本段)
-- -----------------------------------------------------------------------------
ALTER TABLE customers
ADD COLUMN birthday_raw VARCHAR(255) NULL AFTER birthday,
ADD COLUMN report_date_raw VARCHAR(255) NULL AFTER report_date;
-- 备份当前脏值
UPDATE customers SET birthday_raw = birthday, report_date_raw = report_date;
-- -----------------------------------------------------------------------------
-- STEP 2:规范化(从 *_raw 解析,写回 birthday/report_date 为 'YYYY-MM-DD' 或 NULL)
-- 规则:/ . → - ;合并多余 - ;去首尾 - ;8 位连写按 YYYYMMDD 解析;
-- 无法解析者置 NULL(原值仍在 *_raw)。
-- -----------------------------------------------------------------------------
UPDATE customers SET birthday = (
CASE
WHEN birthday_raw IS NULL OR TRIM(birthday_raw) = '' THEN NULL
WHEN birthday_raw REGEXP '^[0-9]{8}$'
THEN DATE_FORMAT(STR_TO_DATE(birthday_raw, '%Y%m%d'), '%Y-%m-%d')
ELSE DATE_FORMAT(
STR_TO_DATE(
TRIM(BOTH '-' FROM
REPLACE(REPLACE(REPLACE(REPLACE(TRIM(birthday_raw),'/','-'),'.','-'),'--','-'),'--','-')
), '%Y-%m-%d'),
'%Y-%m-%d')
END
);
UPDATE customers SET report_date = (
CASE
WHEN report_date_raw IS NULL OR TRIM(report_date_raw) = '' THEN NULL
WHEN report_date_raw REGEXP '^[0-9]{8}$'
THEN DATE_FORMAT(STR_TO_DATE(report_date_raw, '%Y%m%d'), '%Y-%m-%d')
ELSE DATE_FORMAT(
STR_TO_DATE(
TRIM(BOTH '-' FROM
REPLACE(REPLACE(REPLACE(REPLACE(TRIM(report_date_raw),'/','-'),'.','-'),'--','-'),'--','-')
), '%Y-%m-%d'),
'%Y-%m-%d')
END
);
-- -----------------------------------------------------------------------------
-- STEP 3:改列类型 varchar → date(此时列内均为 'YYYY-MM-DD' 或 NULL,转换干净)
-- -----------------------------------------------------------------------------
ALTER TABLE customers MODIFY birthday DATE NULL;
ALTER TABLE customers MODIFY report_date DATE NULL;
-- -----------------------------------------------------------------------------
-- STEP 4:清理离谱年份(仅动可见数据 deleted_at IS NULL;raw 已备份)
-- birthday 合理区间 [1930,2015](清掉 1905 那批 Excel 序列号垃圾等)
-- report_date 合理区间 [2010,2026](清掉 1955 / 5万年 等)
-- 如需连软删除行一并清理,去掉 deleted_at 条件即可。
-- -----------------------------------------------------------------------------
UPDATE customers SET birthday = NULL
WHERE deleted_at IS NULL AND birthday IS NOT NULL
AND (YEAR(birthday) < 1930 OR YEAR(birthday) > 2015);
UPDATE customers SET report_date = NULL
WHERE deleted_at IS NULL AND report_date IS NOT NULL
AND (YEAR(report_date) < 2010 OR YEAR(report_date) > 2026);
-- -----------------------------------------------------------------------------
-- STEP 5:人工抢救可明确还原的 birthday(按原值匹配,id 无关;仅当已置空时)
-- 其余含糊/垃圾值(5 位年、手机号、"美团加购"、1993-27 等)保持 NULL,不臆测。
-- -----------------------------------------------------------------------------
UPDATE customers SET birthday='1985-03-15' WHERE birthday IS NULL AND birthday_raw='1985-0315';
UPDATE customers SET birthday='1987-05-08' WHERE birthday IS NULL AND birthday_raw='1987-5=8';
UPDATE customers SET birthday='1984-01-13' WHERE birthday IS NULL AND birthday_raw='1984-·1-13';
UPDATE customers SET birthday='1982-07-10' WHERE birthday IS NULL AND birthday_raw='1982-7-·10';
-- -----------------------------------------------------------------------------
-- STEP 6:迁移后核对(只读)
-- -----------------------------------------------------------------------------
-- 6.1 类型应为 date
SHOW COLUMNS FROM customers WHERE Field IN ('birthday','report_date','birthday_raw','report_date_raw');
-- 6.2 数量与年份范围(离谱残留应为 0)
SELECT
SUM(birthday IS NOT NULL) AS birthday_有值,
MIN(YEAR(birthday)) AS 生日最小年,
MAX(YEAR(birthday)) AS 生日最大年,
SUM(report_date IS NOT NULL) AS 报单_有值,
MIN(YEAR(report_date)) AS 报单最小年,
MAX(YEAR(report_date)) AS 报单最大年
FROM customers WHERE deleted_at IS NULL;
-- 6.3 未能解析、置空但原值仍在的清单(供人工复核,date 空 + raw 有值)
SELECT id, birthday_raw FROM customers WHERE deleted_at IS NULL AND birthday IS NULL AND TRIM(birthday_raw) <> '';
SELECT id, report_date_raw FROM customers WHERE deleted_at IS NULL AND report_date IS NULL AND TRIM(report_date_raw) <> '';
-- =============================================================================
-- 回滚方案(如迁移后发现问题)
-- A. 值回滚(列类型已改为 date 时):把原始脏值写回不可行(date 存不下脏串),
-- 应从 STEP 0 前的 mysqldump 整表恢复:
-- mysql -h... -ucrm -p crm < crm_backup_YYYY-MM-DD.sql
-- B. 仅想恢复某几行的原始文本:查 *_raw 列即为迁移前原值。
-- ⇒ 因此 STEP 0 的整库备份是回滚的唯一可靠依据,务必先做。
-- =============================================================================