docs: 生产库日期字段迁移+清洗 SQL 运行手册
customers.birthday/report_date 从脏 varchar 迁移为 date 类型, 原值备份到 *_raw 列,清理离谱年份。已在生产真实数据只读验证: birthday 解析 99.9%、report_date 98.9%。含备份/核对/回滚步骤。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,133 @@
|
|||||||
|
-- =============================================================================
|
||||||
|
-- 生产库 crm 日期字段迁移 + 清洗
|
||||||
|
-- 目标:把 customers.birthday / report_date 从脏 varchar 迁移为 date 类型,
|
||||||
|
-- 原值备份到 *_raw 列,并清理离谱年份。全程非破坏(原值可回溯)。
|
||||||
|
--
|
||||||
|
-- 背景:测试库 crm_dev 已是 date 结构;生产库 crm 仍是 varchar 脏数据。
|
||||||
|
-- 本脚本经生产真实数据只读验证:birthday 解析成功率 99.9%、report_date 98.9%,
|
||||||
|
-- 无法解析的原值保留在 *_raw 列,date 置 NULL。
|
||||||
|
--
|
||||||
|
-- ⚠️ 执行前必读:
|
||||||
|
-- 1. 【务必先整库备份】:mysqldump -h124.223.187.95 -ucrm -p crm > crm_backup_$(date +%F).sql
|
||||||
|
-- 2. 建议在业务低峰 / 维护窗口执行;应用可短暂停写。
|
||||||
|
-- 3. 逐段执行,每段后看 STEP 7 的核对结果再继续。
|
||||||
|
-- 4. MySQL 5.7 适用(未用 8.0 的 REGEXP_REPLACE)。
|
||||||
|
-- =============================================================================
|
||||||
|
|
||||||
|
USE crm;
|
||||||
|
|
||||||
|
-- -----------------------------------------------------------------------------
|
||||||
|
-- STEP 0(可选):执行前预览将受影响的数据量(只读,不改数据)
|
||||||
|
-- -----------------------------------------------------------------------------
|
||||||
|
SELECT
|
||||||
|
SUM(birthday IS NOT NULL AND TRIM(birthday)<>'') AS birthday_非空,
|
||||||
|
SUM(report_date IS NOT NULL AND TRIM(report_date)<>'') AS report_非空
|
||||||
|
FROM customers;
|
||||||
|
|
||||||
|
|
||||||
|
-- -----------------------------------------------------------------------------
|
||||||
|
-- STEP 1:新增原值备份列(幂等:若已存在请跳过本段)
|
||||||
|
-- -----------------------------------------------------------------------------
|
||||||
|
ALTER TABLE customers
|
||||||
|
ADD COLUMN birthday_raw VARCHAR(255) NULL AFTER birthday,
|
||||||
|
ADD COLUMN report_date_raw VARCHAR(255) NULL AFTER report_date;
|
||||||
|
|
||||||
|
-- 备份当前脏值
|
||||||
|
UPDATE customers SET birthday_raw = birthday, report_date_raw = report_date;
|
||||||
|
|
||||||
|
|
||||||
|
-- -----------------------------------------------------------------------------
|
||||||
|
-- STEP 2:规范化(从 *_raw 解析,写回 birthday/report_date 为 'YYYY-MM-DD' 或 NULL)
|
||||||
|
-- 规则:/ . → - ;合并多余 - ;去首尾 - ;8 位连写按 YYYYMMDD 解析;
|
||||||
|
-- 无法解析者置 NULL(原值仍在 *_raw)。
|
||||||
|
-- -----------------------------------------------------------------------------
|
||||||
|
UPDATE customers SET birthday = (
|
||||||
|
CASE
|
||||||
|
WHEN birthday_raw IS NULL OR TRIM(birthday_raw) = '' THEN NULL
|
||||||
|
WHEN birthday_raw REGEXP '^[0-9]{8}$'
|
||||||
|
THEN DATE_FORMAT(STR_TO_DATE(birthday_raw, '%Y%m%d'), '%Y-%m-%d')
|
||||||
|
ELSE DATE_FORMAT(
|
||||||
|
STR_TO_DATE(
|
||||||
|
TRIM(BOTH '-' FROM
|
||||||
|
REPLACE(REPLACE(REPLACE(REPLACE(TRIM(birthday_raw),'/','-'),'.','-'),'--','-'),'--','-')
|
||||||
|
), '%Y-%m-%d'),
|
||||||
|
'%Y-%m-%d')
|
||||||
|
END
|
||||||
|
);
|
||||||
|
|
||||||
|
UPDATE customers SET report_date = (
|
||||||
|
CASE
|
||||||
|
WHEN report_date_raw IS NULL OR TRIM(report_date_raw) = '' THEN NULL
|
||||||
|
WHEN report_date_raw REGEXP '^[0-9]{8}$'
|
||||||
|
THEN DATE_FORMAT(STR_TO_DATE(report_date_raw, '%Y%m%d'), '%Y-%m-%d')
|
||||||
|
ELSE DATE_FORMAT(
|
||||||
|
STR_TO_DATE(
|
||||||
|
TRIM(BOTH '-' FROM
|
||||||
|
REPLACE(REPLACE(REPLACE(REPLACE(TRIM(report_date_raw),'/','-'),'.','-'),'--','-'),'--','-')
|
||||||
|
), '%Y-%m-%d'),
|
||||||
|
'%Y-%m-%d')
|
||||||
|
END
|
||||||
|
);
|
||||||
|
|
||||||
|
|
||||||
|
-- -----------------------------------------------------------------------------
|
||||||
|
-- STEP 3:改列类型 varchar → date(此时列内均为 'YYYY-MM-DD' 或 NULL,转换干净)
|
||||||
|
-- -----------------------------------------------------------------------------
|
||||||
|
ALTER TABLE customers MODIFY birthday DATE NULL;
|
||||||
|
ALTER TABLE customers MODIFY report_date DATE NULL;
|
||||||
|
|
||||||
|
|
||||||
|
-- -----------------------------------------------------------------------------
|
||||||
|
-- STEP 4:清理离谱年份(仅动可见数据 deleted_at IS NULL;raw 已备份)
|
||||||
|
-- birthday 合理区间 [1930,2015](清掉 1905 那批 Excel 序列号垃圾等)
|
||||||
|
-- report_date 合理区间 [2010,2026](清掉 1955 / 5万年 等)
|
||||||
|
-- 如需连软删除行一并清理,去掉 deleted_at 条件即可。
|
||||||
|
-- -----------------------------------------------------------------------------
|
||||||
|
UPDATE customers SET birthday = NULL
|
||||||
|
WHERE deleted_at IS NULL AND birthday IS NOT NULL
|
||||||
|
AND (YEAR(birthday) < 1930 OR YEAR(birthday) > 2015);
|
||||||
|
|
||||||
|
UPDATE customers SET report_date = NULL
|
||||||
|
WHERE deleted_at IS NULL AND report_date IS NOT NULL
|
||||||
|
AND (YEAR(report_date) < 2010 OR YEAR(report_date) > 2026);
|
||||||
|
|
||||||
|
|
||||||
|
-- -----------------------------------------------------------------------------
|
||||||
|
-- STEP 5:人工抢救可明确还原的 birthday(按原值匹配,id 无关;仅当已置空时)
|
||||||
|
-- 其余含糊/垃圾值(5 位年、手机号、"美团加购"、1993-27 等)保持 NULL,不臆测。
|
||||||
|
-- -----------------------------------------------------------------------------
|
||||||
|
UPDATE customers SET birthday='1985-03-15' WHERE birthday IS NULL AND birthday_raw='1985-0315';
|
||||||
|
UPDATE customers SET birthday='1987-05-08' WHERE birthday IS NULL AND birthday_raw='1987-5=8';
|
||||||
|
UPDATE customers SET birthday='1984-01-13' WHERE birthday IS NULL AND birthday_raw='1984-·1-13';
|
||||||
|
UPDATE customers SET birthday='1982-07-10' WHERE birthday IS NULL AND birthday_raw='1982-7-·10';
|
||||||
|
|
||||||
|
|
||||||
|
-- -----------------------------------------------------------------------------
|
||||||
|
-- STEP 6:迁移后核对(只读)
|
||||||
|
-- -----------------------------------------------------------------------------
|
||||||
|
-- 6.1 类型应为 date
|
||||||
|
SHOW COLUMNS FROM customers WHERE Field IN ('birthday','report_date','birthday_raw','report_date_raw');
|
||||||
|
|
||||||
|
-- 6.2 数量与年份范围(离谱残留应为 0)
|
||||||
|
SELECT
|
||||||
|
SUM(birthday IS NOT NULL) AS birthday_有值,
|
||||||
|
MIN(YEAR(birthday)) AS 生日最小年,
|
||||||
|
MAX(YEAR(birthday)) AS 生日最大年,
|
||||||
|
SUM(report_date IS NOT NULL) AS 报单_有值,
|
||||||
|
MIN(YEAR(report_date)) AS 报单最小年,
|
||||||
|
MAX(YEAR(report_date)) AS 报单最大年
|
||||||
|
FROM customers WHERE deleted_at IS NULL;
|
||||||
|
|
||||||
|
-- 6.3 未能解析、置空但原值仍在的清单(供人工复核,date 空 + raw 有值)
|
||||||
|
SELECT id, birthday_raw FROM customers WHERE deleted_at IS NULL AND birthday IS NULL AND TRIM(birthday_raw) <> '';
|
||||||
|
SELECT id, report_date_raw FROM customers WHERE deleted_at IS NULL AND report_date IS NULL AND TRIM(report_date_raw) <> '';
|
||||||
|
|
||||||
|
|
||||||
|
-- =============================================================================
|
||||||
|
-- 回滚方案(如迁移后发现问题)
|
||||||
|
-- A. 值回滚(列类型已改为 date 时):把原始脏值写回不可行(date 存不下脏串),
|
||||||
|
-- 应从 STEP 0 前的 mysqldump 整表恢复:
|
||||||
|
-- mysql -h... -ucrm -p crm < crm_backup_YYYY-MM-DD.sql
|
||||||
|
-- B. 仅想恢复某几行的原始文本:查 *_raw 列即为迁移前原值。
|
||||||
|
-- ⇒ 因此 STEP 0 的整库备份是回滚的唯一可靠依据,务必先做。
|
||||||
|
-- =============================================================================
|
||||||
Reference in New Issue
Block a user