Detecting utf8 broken characters in MySQL

Thanks for your answers!!

I fixed my tables with this, and wanted to share the full list of changes. Note that it also includes fixing html decoded characters, besides latin ones, it was really a mess:

(If you need more conversions, look them up at https://www.utf8-chartable.de/unicode-utf8-table.pl)

update `table` set `field` = replace(`field`, 'â€', '"');
update `table` set `field` = replace(`field`, 'â€“', '–');
update `table` set `field` = replace(`field`, 'â€¢', '-');
update `table` set `field` = replace(`field`, 'â€œ', '"');

update `table` set `field` = replace(`field`, 'Â¡', '¡');
update `table` set `field` = replace(`field`, 'Â¢', '¢');
update `table` set `field` = replace(`field`, 'Â£', '£');
update `table` set `field` = replace(`field`, 'Â¤', '¤');
update `table` set `field` = replace(`field`, 'Â¥', '¥');
update `table` set `field` = replace(`field`, 'Â¦', '¦');
update `table` set `field` = replace(`field`, 'Â§', '§');
update `table` set `field` = replace(`field`, 'Â¨', '¨');
update `table` set `field` = replace(`field`, 'Â©', '©');
update `table` set `field` = replace(`field`, 'Âª', 'ª');
update `table` set `field` = replace(`field`, 'Â«', '«');
update `table` set `field` = replace(`field`, 'Â¬', '¬');
# This one looks like it's missing a character, but it's there. 0xad
update `table` set `field` = replace(`field`, 'Â', '');
update `table` set `field` = replace(`field`, 'Â®', '®');
update `table` set `field` = replace(`field`, 'Â¯', '¯');
update `table` set `field` = replace(`field`, 'Â°', '°');
update `table` set `field` = replace(`field`, 'Â±', '±');
update `table` set `field` = replace(`field`, 'Â²', '²');
update `table` set `field` = replace(`field`, 'Â³', '³');
update `table` set `field` = replace(`field`, 'Â´', '´');
update `table` set `field` = replace(`field`, 'Âµ', 'µ');
update `table` set `field` = replace(`field`, 'Â¶', '¶');
update `table` set `field` = replace(`field`, 'Â·', '·');
update `table` set `field` = replace(`field`, 'Â¸', '¸');
update `table` set `field` = replace(`field`, 'Â¹', '¹');
update `table` set `field` = replace(`field`, 'Âº', 'º');
update `table` set `field` = replace(`field`, 'Â»', '»');
update `table` set `field` = replace(`field`, 'Â¼', '¼');
update `table` set `field` = replace(`field`, 'Â½', '½');
update `table` set `field` = replace(`field`, 'Â¾', '¾');
update `table` set `field` = replace(`field`, 'Â¿', '¿');

update `table` set `field` = replace(`field`, 'Ã€', 'À');
# This one looks like it's missing a character, but it's there. 0x81
update `table` set `field` = replace(`field`, 'Ã', 'Á');
update `table` set `field` = replace(`field`, 'Ã‚', 'Â');
update `table` set `field` = replace(`field`, 'Ãƒ', 'Ã');
update `table` set `field` = replace(`field`, 'Ã„', 'Ä');
update `table` set `field` = replace(`field`, 'Ã…', 'Å');
update `table` set `field` = replace(`field`, 'Ã†', 'Æ');
update `table` set `field` = replace(`field`, 'Ã‡', 'Ç');
update `table` set `field` = replace(`field`, 'Ãˆ', 'È');
update `table` set `field` = replace(`field`, 'Ã‰', 'É');
update `table` set `field` = replace(`field`, 'ÃŠ', 'Ê');
update `table` set `field` = replace(`field`, 'Ã‹', 'Ë');
update `table` set `field` = replace(`field`, 'ÃŒ', 'Ì');
# This one looks like it's missing a character, but it's there. 0x8d
update `table` set `field` = replace(`field`, 'Ã', 'Í');
update `table` set `field` = replace(`field`, 'ÃŽ', 'Î');
# This one looks like it's missing a character, but it's there. 0x8f
update `table` set `field` = replace(`field`, 'Ã', 'Ï');
# This one looks like it's missing a character, but it's there. 0x90
update `table` set `field` = replace(`field`, 'Ã', 'Ð');
update `table` set `field` = replace(`field`, 'Ã‘', 'Ñ');
update `table` set `field` = replace(`field`, 'Ã’', 'Ò');
update `table` set `field` = replace(`field`, 'Ã“', 'Ó');
update `table` set `field` = replace(`field`, 'Ã”', 'Ô');
update `table` set `field` = replace(`field`, 'Ã•', 'Õ');
update `table` set `field` = replace(`field`, 'Ã–', 'Ö');
update `table` set `field` = replace(`field`, 'Ã—', '×');
update `table` set `field` = replace(`field`, 'Ã˜', 'Ø');
update `table` set `field` = replace(`field`, 'Ã™', 'Ù');
update `table` set `field` = replace(`field`, 'Ãš', 'Ú');
update `table` set `field` = replace(`field`, 'Ã›', 'Û');
update `table` set `field` = replace(`field`, 'Ãœ', 'Ü');
# This one looks like it's missing a character, but it's there. 0x9d
update `table` set `field` = replace(`field`, 'Ã', 'Ý');
update `table` set `field` = replace(`field`, 'Ãž', 'Þ');
update `table` set `field` = replace(`field`, 'ÃŸ', 'ß');
update `table` set `field` = replace(`field`, 'Ã ', 'à');
update `table` set `field` = replace(`field`, 'Ã¡', 'á');
update `table` set `field` = replace(`field`, 'Ã¢', 'â');
update `table` set `field` = replace(`field`, 'Ã£', 'ã');
update `table` set `field` = replace(`field`, 'Ã¤', 'ä');
update `table` set `field` = replace(`field`, 'Ã¥', 'å');
update `table` set `field` = replace(`field`, 'Ã¦', 'æ');
update `table` set `field` = replace(`field`, 'Ã§', 'ç');
update `table` set `field` = replace(`field`, 'Ã¨', 'è');
update `table` set `field` = replace(`field`, 'Ã©', 'é');
update `table` set `field` = replace(`field`, 'Ãª', 'ê');
update `table` set `field` = replace(`field`, 'Ã«', 'ë');
update `table` set `field` = replace(`field`, 'Ã¬', 'ì');
# This one looks like it's missing a character, but it's there. 0xad
update `table` set `field` = replace(`field`, 'Ã', 'í');
update `table` set `field` = replace(`field`, 'Ã®', 'î');
update `table` set `field` = replace(`field`, 'Ã¯', 'ï');
update `table` set `field` = replace(`field`, 'Ã°', 'ð');
update `table` set `field` = replace(`field`, 'Ã±', 'ñ');
update `table` set `field` = replace(`field`, 'Ã²', 'ò');
update `table` set `field` = replace(`field`, 'Ã³', 'ó');
update `table` set `field` = replace(`field`, 'Ã´', 'ô');
update `table` set `field` = replace(`field`, 'Ãµ', 'õ');
update `table` set `field` = replace(`field`, 'Ã¶', 'ö');
update `table` set `field` = replace(`field`, 'Ã·', '÷');
update `table` set `field` = replace(`field`, 'Ã¸', 'ø');
update `table` set `field` = replace(`field`, 'Ã¹', 'ù');
update `table` set `field` = replace(`field`, 'Ãº', 'ú');
update `table` set `field` = replace(`field`, 'Ã»', 'û');
update `table` set `field` = replace(`field`, 'Ã¼', 'ü');
update `table` set `field` = replace(`field`, 'Ã½', 'ý');
update `table` set `field` = replace(`field`, 'Ã¾', 'þ');
update `table` set `field` = replace(`field`, 'Ã¿', 'ÿ');

update `table` set `field` = replace(`field` ,'&ccedil;','ç');
update `table` set `field` = replace(`field` ,'&atilde;','ã');
update `table` set `field` = replace(`field` ,'&aacute;','á');
update `table` set `field` = replace(`field` ,'&acirc;','â');
update `table` set `field` = replace(`field` ,'&eacute;','é');
update `table` set `field` = replace(`field` ,'&iacute;','í');
update `table` set `field` = replace(`field` ,'&otilde;','õ');
update `table` set `field` = replace(`field` ,'&uacute;','ú');
update `table` set `field` = replace(`field` ,'&ccedil;','ç');
update `table` set `field` = replace(`field` ,'&Aacute;','Á');
update `table` set `field` = replace(`field` ,'&Acirc;','Â');
update `table` set `field` = replace(`field` ,'&Eacute;','É');
update `table` set `field` = replace(`field` ,'&Iacute;','Í');
update `table` set `field` = replace(`field` ,'&Otilde;','Õ');
update `table` set `field` = replace(`field` ,'&Uacute;','Ú');
update `table` set `field` = replace(`field` ,'&Ccedil;','Ç');
update `table` set `field` = replace(`field` ,'&Atilde;','Ã');
update `table` set `field` = replace(`field` ,'&Agrave;','À');
update `table` set `field` = replace(`field` ,'&Ecirc;','Ê');
update `table` set `field` = replace(`field` ,'&Oacute;','Ó');
update `table` set `field` = replace(`field` ,'&Ocirc;','Ô');
update `table` set `field` = replace(`field` ,'&Uuml;','Ü');
update `table` set `field` = replace(`field` ,'&atilde;','ã');
update `table` set `field` = replace(`field` ,'&agrave;','à');
update `table` set `field` = replace(`field` ,'&ecirc;','ê');
update `table` set `field` = replace(`field` ,'&oacute;','ó');
update `table` set `field` = replace(`field` ,'&ocirc;','ô');
update `table` set `field` = replace(`field` ,'&uuml;','ü');
update `table` set `field` = replace(`field` ,'&amp;','&');
update `table` set `field` = replace(`field` ,'&gt;','>');
update `table` set `field` = replace(`field` ,'&lt;','<');
update `table` set `field` = replace(`field` ,'&circ;','ˆ');
update `table` set `field` = replace(`field` ,'&tilde;','˜');
update `table` set `field` = replace(`field` ,'&uml;','¨');
update `table` set `field` = replace(`field` ,'&cute;','´');
update `table` set `field` = replace(`field` ,'&cedil;','¸');
update `table` set `field` = replace(`field` ,'&quot;','"');
update `table` set `field` = replace(`field` ,'&ldquo;','“');
update `table` set `field` = replace(`field` ,'&rdquo;','”');
update `table` set `field` = replace(`field` ,'&lsquo;','‘');
update `table` set `field` = replace(`field` ,'&rsquo;','’');
update `table` set `field` = replace(`field` ,'&lsaquo;','‹');
update `table` set `field` = replace(`field` ,'&rsaquo;','›');
update `table` set `field` = replace(`field` ,'&laquo;','«');
update `table` set `field` = replace(`field` ,'&raquo;','»');
update `table` set `field` = replace(`field` ,'&ordm;','º');
update `table` set `field` = replace(`field` ,'&ordf;','ª');
update `table` set `field` = replace(`field` ,'&ndash;','–');
update `table` set `field` = replace(`field` ,'&mdash;','—');
update `table` set `field` = replace(`field` ,'&macr;','¯');
update `table` set `field` = replace(`field` ,'&hellip;','…');
update `table` set `field` = replace(`field` ,'&brvbar;','¦');
update `table` set `field` = replace(`field` ,'&bull;','•');
update `table` set `field` = replace(`field` ,'&para;','¶');
update `table` set `field` = replace(`field` ,'&sect;','§');
update `table` set `field` = replace(`field` ,'&sup1;','¹');
update `table` set `field` = replace(`field` ,'&sup2;','²');
update `table` set `field` = replace(`field` ,'&sup3;','³');
update `table` set `field` = replace(`field` ,'&frac12;','½');
update `table` set `field` = replace(`field` ,'&frac14;','¼');
update `table` set `field` = replace(`field` ,'&frac34;','¾');
update `table` set `field` = replace(`field` ,'&#8539;','⅛');
update `table` set `field` = replace(`field` ,'&#8540;','⅜');
update `table` set `field` = replace(`field` ,'&#8541;','⅝');
update `table` set `field` = replace(`field` ,'&#8542;','⅞');
update `table` set `field` = replace(`field` ,'&gt;','>');
update `table` set `field` = replace(`field` ,'&lt;','<');
update `table` set `field` = replace(`field` ,'&plusmn;','±');
update `table` set `field` = replace(`field` ,'&minus;','−');
update `table` set `field` = replace(`field` ,'&times;','×');
update `table` set `field` = replace(`field` ,'&divide;','÷');
update `table` set `field` = replace(`field` ,'&lowast;','∗');
update `table` set `field` = replace(`field` ,'&frasl;','⁄');
update `table` set `field` = replace(`field` ,'&permil;','‰');
update `table` set `field` = replace(`field` ,'&int;','∫');
update `table` set `field` = replace(`field` ,'&sum;','∑');
update `table` set `field` = replace(`field` ,'&prod;','∏');
update `table` set `field` = replace(`field` ,'&radic;','√');
update `table` set `field` = replace(`field` ,'&infin;','∞');
update `table` set `field` = replace(`field` ,'&asymp;','≈');
update `table` set `field` = replace(`field` ,'&cong;','≅');
update `table` set `field` = replace(`field` ,'&prop;','∝');
update `table` set `field` = replace(`field` ,'&equiv;','≡');
update `table` set `field` = replace(`field` ,'&ne;','≠');
update `table` set `field` = replace(`field` ,'&le;','≤');
update `table` set `field` = replace(`field` ,'&ge;','≥');
update `table` set `field` = replace(`field` ,'&there4;','∴');
update `table` set `field` = replace(`field` ,'&sdot;','⋅');
update `table` set `field` = replace(`field` ,'&middot;','·');
update `table` set `field` = replace(`field` ,'&part;','∂');
update `table` set `field` = replace(`field` ,'&image;','ℑ');
update `table` set `field` = replace(`field` ,'&real;','ℜ');
update `table` set `field` = replace(`field` ,'&prime;','′');
update `table` set `field` = replace(`field` ,'&Prime;','″');
update `table` set `field` = replace(`field` ,'&deg;','°');
update `table` set `field` = replace(`field` ,'&ang;','∠');
update `table` set `field` = replace(`field` ,'&perp;','⊥');
update `table` set `field` = replace(`field` ,'&nabla;','∇');
update `table` set `field` = replace(`field` ,'&oplus;','⊕');
update `table` set `field` = replace(`field` ,'&otimes;','⊗');
update `table` set `field` = replace(`field` ,'&alefsym;','ℵ');
update `table` set `field` = replace(`field` ,'&oslash;','ø');
update `table` set `field` = replace(`field` ,'&Oslash;','Ø');
update `table` set `field` = replace(`field` ,'&isin;','∈');
update `table` set `field` = replace(`field` ,'&notin;','∉');
update `table` set `field` = replace(`field` ,'&cap;','∩');
update `table` set `field` = replace(`field` ,'&cup;','∪');
update `table` set `field` = replace(`field` ,'&sub;','⊂');
update `table` set `field` = replace(`field` ,'&sup;','⊃');
update `table` set `field` = replace(`field` ,'&sube;','⊆');
update `table` set `field` = replace(`field` ,'&supe;','⊇');
update `table` set `field` = replace(`field` ,'&exist;','∃');
update `table` set `field` = replace(`field` ,'&forall;','∀');
update `table` set `field` = replace(`field` ,'&empty;','∅');
update `table` set `field` = replace(`field` ,'&not;','¬');
update `table` set `field` = replace(`field` ,'&and;','∧');
update `table` set `field` = replace(`field` ,'&or;','∨');
update `table` set `field` = replace(`field` ,'&crarr;','↵');

I fixed with

UPDATE wp_zcs9ck_posts_copy SET post_title = 
    CONVERT(BINARY CONVERT(post_title USING latin1) USING utf8);

Complete solution: http://jonisalonen.com/2012/fixing-doubly-utf-8-encoded-text-in-mysql/

UPDATE `table_name` SET `column_name` = REPLACE(`column_name` ,'Ã¡','á');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name` ,'Ã¤','ä');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name` ,'Ã©','é');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name` ,'í©','é');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name` ,'Ã³','ó');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name` ,'íº','ú');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name` ,'Ãº','ú');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name` ,'Ã±','ñ');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name` ,'í‘','Ñ');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name` ,'Ã','í');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name` ,'â€“','–');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name`,'â€™','\'');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name`,'â€¦','...');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name`,'â€“','-');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name`,'â€œ','"');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name`,'â€','"');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name`,'â€˜','\'');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name`,'â€¢','-');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name`,'â€¡','c');
UPDATE `table_name` SET `column_name` = REPLACE(`column_name` ,'Â','');

The SELECT statement you need is the following:

SELECT * FROM TABLE WHERE LENGTH(name) != CHAR_LENGTH(name);

This returns all rows which contain multi-byte characters.

name is assumed to be a field / the field where weird characters would be found. *

Detecting utf8 broken characters in MySQL

Tags:

Mysql

Utf 8

Related

Recent Posts