Fixing input charset issues by autodetecting

.gitignore

@@ -55,3 +55,6 @@ docs/_build/
 
 # PyBuilder
 target/
+
+# Ignore the in here installed virtualenv
+virtualenv/

README.md

@@ -24,6 +24,9 @@ Used python3 modules:
 - `email` to construct the email.
 - `syslog` for logging the sending of emails into syslog
 
+External modules:
+- `chardet`, to autodetect the input charset (which can vary)
+
 ## Requirements:
 
 - A mailserver.
@@ -35,7 +38,9 @@ Used python3 modules:
 
 - Install `python3` (do I need to say this?)
 - Clone this repository into a random directory, reachable for dovecot2
-- Initialize the sqlite db with running `init-schema.sh`
+- Change to that directory
+- Install virtualenv and external modules by running `tools/install.sh`
+- Initialize the sqlite db with running `tools/init-schema.sh`
 - Copy your replies (mentioned above) into the same repo directory
 - Edit your sieve configuration to pipe your given role into this tool.
 

email-responder.py

@@ -2,12 +2,12 @@
 
 import sys
 import os
-import io
 import smtplib
 import sqlite3
 import datetime
 import syslog
 import codecs
+import chardet
 
 from email.mime.text import MIMEText
 from email.mime.multipart import MIMEMultipart
@@ -19,8 +19,12 @@ from email.header import decode_header, make_header
 my_dir = os.path.dirname(os.path.realpath(__file__))
 os.chdir(my_dir)
 
-stdin_utf8 = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8')
-original_headers = Parser().parsestr(stdin_utf8.read())
+input_bytes = sys.stdin.buffer.read()  # This is bytes
+
+encoding_result = chardet.detect(input_bytes)
+input_decoded = input_bytes.decode(encoding_result['encoding'])
+
+original_headers = Parser().parsestr(input_decoded)
 # with open('email1.txt') as fp:
 #     original_headers = Parser().parsestr(fp.read())
 

email-responder.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+PROJECT_DIR=$(cd $(dirname ${BASH_SOURCE[0]});pwd)
+
+cd $PROJECT_DIR
+
+source virtualenv/bin/activate
+
+exec python3 email-responder.py

requirements.txt

@@ -0,0 +1 @@
+chardet==2.3.0

tools/install.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+PROJECT_DIR=$(cd $(dirname ${BASH_SOURCE[0]});cd ..;pwd)
+
+cd $PROJECT_DIR
+
+pyvenv virtualenv
+source virtualenv/bin/activate
+
+pip install -r requirements.txt