当遇到某人发来的文本內容内不是标准 utf-8 编码时，可以使用本程序来进行转换。

转换时遇到文本行尾使用了Windows专用的 CRLF 换行符时，也会将文本行尾统一更换为 LF。这样的好处是可以缩小文本文件的空间占用，并且可以保证该文件可以在基于Unix和Linux的操作系统以相同的格式显示。

encoding_line_ending_converter.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import argparse
import os


def main():
    parser = argparse.ArgumentParser(description='Text Encoding and Line Ending Converter')
    parser.add_argument('path', type=str, help='directory to be traversed and processed.')
    parser.add_argument('encoding', type=str, help='encoding of the text file.')
    parser.add_argument('-e', '--extensions', type=str, help='only process specified file extensions. use "|" as the separator, for example: ".txt|.log|.html"', default='')
    parser.add_argument('-q', '--quiet', action='store_true', help='do not output information during processing.')
    parser.add_argument("-v", "--verbose", action="store_true", help="increase output verbosity")
    args = parser.parse_args()
    extensions = tuple(args.extensions.split('|'))

    processed_count = 0
    for root, _, files in os.walk(args.path):
        for file_name in files:
            if len(extensions) == 0 or file_name.endswith(extensions):
                if trans(os.path.join(root, file_name), args.encoding, args.quiet, args.verbose):
                    processed_count += 1

    if not args.quiet:
        print(f'{processed_count} files have been processed.')


def trans(file_path, file_encoding, quiet, verbose) -> bool:
    """Convert file encoding and line endings."""
    # check file encoding
    encoding = 'utf-8'
    content = ''
    try:
        with open(file_path, 'r', encoding=encoding) as f:
            content = f.read()
    except UnicodeDecodeError:
        with open(file_path, 'rb') as f:
            try:
                content = f.read().decode(file_encoding)
                encoding = file_encoding
            except UnicodeDecodeError:
                if not quiet:
                    print(f'{file_path} the file encoding is not {file_encoding}.')
                return False

    # determine if a file uses LF line endings.
    if '\r' in content:
        # convert CRLF line endings to LF.
        content = content.replace('\r\n', '\n')

        # write content
        with open(file_path, 'w', encoding='utf-8', newline='\n') as f:
            f.write(content)

        if not quiet:
            print(f'{file_path} the file encoding and line endings have been converted to UTF-8 and LF.')
        return True
    if encoding != 'utf-8':
        # write content
        with open(file_path, 'w', encoding='utf-8', newline='\n') as f:
            f.write(content)

        if not quiet:
            print(f'{file_path} the file encoding has been converted to UTF-8.')
        return True
    if not quiet and verbose:
        print(f'{file_path} the file is now encoded in UTF-8 with LF line endings.')
    return False


if __name__ == '__main__':
    main()

点我下载：encoding_line_ending_converter.py

> python encoding_line_ending_converter.py -h

usage: encoding_line_ending_converter.py [-h] [-e EXTENSIONS] [-q] [-v] path encoding

Text Encoding and Line Ending Converter

positional arguments:
  path                  directory to be traversed and processed.
  encoding              encoding of the text file.

options:
  -h, --help            show this help message and exit
  -e EXTENSIONS, --extensions EXTENSIONS
                        only process specified file extensions. use "|" as the separator, for example: ".txt|.log|.html"
  -q, --quiet           do not output information during processing.
  -v, --verbose         increase output verbosity

其中 encoding 就是python中常用的编码代码，例如： gbk、cp437 等等…

RSS 订阅