EAI support for notqmail

.cirrus.yml

@@ -12,9 +12,10 @@ task:
  - MAKE_FLAGS: "-j 2 NROFF=true"
 
  pkginstall_script:
- - pkg install -y groff pkgconf check
+ - pkg install -y groff pkgconf check libidn2
  configure_script:
- - echo "clang -O2 -Wall -Wshadow -Werror=implicit-function-declaration -Werror=deprecated-declarations" > conf-cc
+ - echo "clang -O2 -Wall -I/usr/local/include -Wshadow -Werror=implicit-function-declaration -Werror=deprecated-declarations" > conf-cc
+ - echo "cc -s -L/usr/local/lib" > conf-ld
  compile_script:
  - make ${MAKE_FLAGS} it man
  test_script:

.github/workflows/ccpp.yml

@@ -70,7 +70,7 @@ jobs:
  uses: actions/checkout@v2
 
  - name: test_deps
- run: if [ "${OS}" = "macos-latest" ]; then brew install check; else sudo apt-get install check; fi
+ run: if [ "${OS}" = "macos-latest" ]; then brew install check; else sudo apt-get install check libidn2-0-dev; fi
  env:
  OS: ${{ matrix.host }}
 

Makefile

@@ -1362,16 +1362,37 @@ alloc.h substdio.h datetime.h now.h datetime.h triggerpull.h extra.h \
 uidgid.h auto_qmail.h auto_uids.h auto_users.h date822fmt.h fmtqfn.h
  ./compile qmail-queue.c
 
+# use tryidn21 for idn2.lib instead of tryidn2 else parallel compile race condition will
+# break things for idn2.lib
+hassmtputf8.h: \
+tryidn2.c compile load conf-smtputf8
+ ( ( ./compile `grep -h -v "^#" conf-smtputf8` tryidn2.c \
+ && ./load tryidn2 -lidn2 ) >/dev/null 2>&1 \
+ && echo \#define SMTPUTF8 1 || echo "WARNING!!! Not compiled with -DSMTPUTF8" 1<&2 ) > hassmtputf8.h
+ rm -f tryidn2.o tryidn2
+
+idn2.lib: \
+tryidn2.c compile load conf-smtputf8
+ ( ( ./compile `grep -h -v "^#" conf-smtputf8` tryidn2.c -o tryidn21.o \
+ && ./load tryidn21 -lidn2 ) >/dev/null 2>&1 \
+ && echo "-lidn2" || echo "WARNING!!! Not linked with libidn2" 1>&2 ) > idn2.lib
+ rm -f tryidn21.o tryidn21
+
+utf8read.o: \
+compile utf8read.c hassmtputf8.h stralloc.h case.h substdio.h subfd.h
+ ./compile utf8read.c
+
 qmail-remote: \
 load qmail-remote.o control.o constmap.o timeoutread.o timeoutwrite.o \
-timeoutconn.o tcpto.o dns.o ip.o ipalloc.o ipme.o quote.o \
-ndelay.a case.a sig.a open.a lock.a getln.a stralloc.a \
-substdio.a error.a str.a fs.a auto_qmail.o dns.lib socket.lib
+timeoutconn.o tcpto.o dns.o ip.o ipalloc.o ipme.o quote.o envread.o \
+utf8read.o ndelay.a case.a sig.a open.a lock.a getln.a stralloc.a \
+substdio.a error.a str.a fs.a auto_qmail.o dns.lib socket.lib \
+idn2.lib
  ./load qmail-remote control.o constmap.o timeoutread.o \
- timeoutwrite.o timeoutconn.o tcpto.o dns.o ip.o \
+ timeoutwrite.o timeoutconn.o tcpto.o dns.o ip.o utf8read.o \
  ipalloc.o ipme.o quote.o ndelay.a case.a sig.a open.a \
- lock.a getln.a stralloc.a substdio.a error.a \
- str.a fs.a auto_qmail.o `cat dns.lib` `cat socket.lib`
+ envread.o lock.a getln.a stralloc.a substdio.a error.a \
+ str.a fs.a auto_qmail.o `cat dns.lib socket.lib idn2.lib`
 
 qmail-remote.0: \
 qmail-remote.8
@@ -1382,7 +1403,7 @@ subfd.h substdio.h scan.h case.h error.h auto_qmail.h control.h dns.h \
 alloc.h quote.h ip.h ipalloc.h ip.h gen_alloc.h ipme.h ip.h ipalloc.h \
 gen_alloc.h gen_allocdefs.h str.h now.h datetime.h exit.h constmap.h \
 tcpto.h readwrite.h timeoutconn.h timeoutread.h timeoutwrite.h oflops.h \
-error.h
+error.h hassmtputf8.h utf8read.h env.h
  ./compile qmail-remote.c
 
 qmail-rspawn: \

TARGETS

@@ -374,3 +374,7 @@ forgeries.0
 setup
 qtmp.h
 qmail-send.service
+trysmtputf8
+idn2.lib
+hassmtputf8.h
+utf8read.o

conf-smtputf8

@@ -0,0 +1 @@
+-DSMTPUTF8

qmail-control.9

@@ -48,6 +48,8 @@ control default used by
 .I defaultdomain \fIme \fRqmail-inject
 .I defaulthost \fIme \fRqmail-inject
 .I databytes \fR0 \fRqmail-smtpd
+.I smtputf8 \fR0 \fRqmail-smtpd
+.I smtputf8 \fR0 \fRqmail-remote
 .I doublebouncehost \fIme \fRqmail-send
 .I doublebounceto \fRpostmaster \fRqmail-send
 .I envnoathost \fIme \fRqmail-send

qmail-remote.8

@@ -47,6 +47,14 @@ arguments to
 The envelope sender address is listed as
 .I sender\fP.
 
+If the environment variable SMTPUTF8 is defined,
+.B qmail-remote
+will respect SMTPUTF8 and EAI addresses. If message is utf8,
+.B qmail-remote
+will use idn2_lookup_u8(3) to perform IDNA2008 lookup string conversion
+on
+.IR host .
+
 Note that
 .B qmail-remote
 does not take options
@@ -202,6 +210,11 @@ Number of seconds
 .B qmail-remote
 will wait for each response from the remote SMTP server.
 Default: 1200.
+.TP 5
+.I smtputf8
+Non-zero value in this control file enables
+.B qmail-remote
+to check if remote SMTP supports RFC6531 SMTP Extension for Internationalized Email. 
 .SH "SEE ALSO"
 addresses(5),
 envelopes(5),

qmail-remote.c

@@ -27,6 +27,12 @@
 #include "timeoutconn.h"
 #include "timeoutread.h"
 #include "timeoutwrite.h"
+#include "hassmtputf8.h"
+#ifdef SMTPUTF8
+#include <idn2.h>
+#endif
+#include "utf8read.h"
+#include "env.h"
 
 #define HUGESMTPTEXT 5000
 
@@ -47,6 +53,12 @@ saa reciplist = {0};
 
 struct ip_address partner;
 
+static stralloc idnhost = { 0 };
+static int smtputf8 = 0; /*- if remote has SMTPUTF8 capability */
+/* set by control file control/smtputf8. zero if SMTPUTF8 not #defined */
+static int enable_smtputf8; 
+int flagutf8; /*- if sender, recipient, received headers have utf8 */
+
 void out(s) char *s; { if (substdio_puts(subfdoutsmall,s) == -1) _exit(0); }
 void zero() { if (substdio_put(subfdoutsmall,"\0",1) == -1) _exit(0); }
 void zerodie() { zero(); substdio_flush(subfdoutsmall); _exit(0); }
@@ -122,32 +134,46 @@ static void get(unsigned char *uc)
 {
  char *ch = (char *)uc;
  substdio_get(&smtpfrom,ch,1);
- if (*ch != '\r')
- if (smtptext.len < HUGESMTPTEXT)
- if (!stralloc_append(&smtptext,ch)) temp_nomem();
+ if (*ch != '\r' && smtptext.len < HUGESMTPTEXT &&
+ !stralloc_append(&smtptext,ch))
+ temp_nomem();
+}
+
+static unsigned long get3()
+{
+ char str[4];
+ int i;
+ unsigned long code;
+
+ substdio_get(&smtpfrom,str,3);
+ str[3] = 0;
+ for (i = 0;i < 3;i++) {
+ if (str[i] == '\r') continue;
+ if (smtptext.len < HUGESMTPTEXT &&
+ !stralloc_append(&smtptext,str+i))
+ temp_nomem();
+ }
+ scan_ulong(str,&code);
+ return code;
 }
 
 unsigned long smtpcode()
 {
- unsigned char ch;
- unsigned long code;
+ unsigned char ch;
+ unsigned long code;
+ int err = 0;
 
  if (!stralloc_copys(&smtptext,"")) temp_nomem();
-
- get(&ch); code = ch - '0';
- get(&ch); code = code * 10 + (ch - '0');
- get(&ch); code = code * 10 + (ch - '0');
+ if ((code = get3()) < 200) err = 1;
  for (;;) {
- get(&ch);
+ get((char *)&ch);
+ if (ch != ' ' && ch != '-') err = 1;
  if (ch != '-') break;
- while (ch != '\n') get(&ch);
- get(&ch);
- get(&ch);
- get(&ch);
+ while (ch != '\n') get((char *)&ch);
+ if (get3() != code) err = 1;
  }
- while (ch != '\n') get(&ch);
-
- return code;
+ while (ch != '\n') get((char *)&ch);
+ return err ? 400 : code;
 }
 
 void outsmtptext()
@@ -211,6 +237,33 @@ void blast()
  substdio_flush(&smtpto);
 }
 
+#ifdef SMTPUTF8
+/*
+ * this function is general purpose
+ * we could use it when we add AUTH,
+ * STARTTLS, SIZE extensions
+ */
+int get_capability(const char *capa)
+{
+ int i = 0, len;
+
+ /* e.g.
+ * 250-PIPELINING
+ * 250-8BITMIME
+ * 250-SIZE 10000000
+ * 250-ETRN
+ * 250-STARTTLS
+ * 250-SMTPUTF8
+ * 250 HELP
+ */
+ len = str_len(capa);
+ for (i = 0; i < smtptext.len-len; ++i)
+ if (case_starts(smtptext.s+i,capa)) return 1;
+
+ return 0;
+}
+#endif
+
 stralloc recip = {0};
 
 void smtp()
@@ -221,15 +274,49 @@ void smtp()
 
  if (smtpcode() != 220) quit("ZConnected to "," but greeting failed");
 
+#ifdef SMTPUTF8
+ /*-
+ * this part of the code is general purpose
+ * it can be used for
+ * checking other extensions like
+ * AUTH, STARTTLS, SIZE, etc
+ */
+ substdio_puts(&smtpto,"EHLO ");
+ substdio_put(&smtpto,helohost.s,helohost.len);
+ substdio_puts(&smtpto,"\r\n");
+ substdio_flush(&smtpto);
+
+ if (smtpcode() != 250) { /*- do a HELO if EHLO fails */
+ substdio_puts(&smtpto,"HELO ");
+ substdio_put(&smtpto,helohost.s,helohost.len);
+ substdio_puts(&smtpto,"\r\n");
+ substdio_flush(&smtpto);
+
+ code = smtpcode();
+ if (code >= 500) quit("DConnected to "," but my name was rejected");
+ if (code != 250) quit("ZConnected to "," but my name was rejected");
+ } else /* EHLO succeeded. Let's check SMTPUTF8 capa */
+ if (enable_smtputf8) {
+ smtputf8 = get_capability("SMTPUTF8"); /*- did the remote server advertize SMTPUTF8 */
+ if (!flagutf8)
+ flagutf8 = utf8read();
+ if (flagutf8 && !smtputf8)
+ quit("DConnected to "," but server does not support internationalized email addresses");
+ }
+#else
  substdio_puts(&smtpto,"HELO ");
  substdio_put(&smtpto,helohost.s,helohost.len);
  substdio_puts(&smtpto,"\r\n");
  substdio_flush(&smtpto);
  if (smtpcode() != 250) quit("ZConnected to "," but my name was rejected");
+#endif
 
  substdio_puts(&smtpto,"MAIL FROM:<");
  substdio_put(&smtpto,sender.s,sender.len);
- substdio_puts(&smtpto,">\r\n");
+ if (enable_smtputf8 && flagutf8)
+ substdio_puts(&smtpto,"> SMTPUTF8\r\n");
+ else
+ substdio_puts(&smtpto,">\r\n");
  substdio_flush(&smtpto);
  code = smtpcode();
  if (code >= 500) quit("DConnected to "," but sender was rejected");
@@ -262,6 +349,7 @@ void smtp()
  if (code >= 500) quit("D"," failed on DATA command");
  if (code >= 400) quit("Z"," failed on DATA command");
 
+ if (enable_smtputf8 && header.len) substdio_put(&smtpto, header.s, header.len);
  blast();
  code = smtpcode();
  flagcritical = 0;
@@ -276,7 +364,10 @@ stralloc canonbox = {0};
 void addrmangle(stralloc *saout, char *s)
 {
  int j;
-
+
+ if (enable_smtputf8 && !flagutf8)
+ flagutf8 = containsutf8(s,str_len(s));
+
  j = str_rchr(s,'@');
  if (!s[j]) {
  if (!stralloc_copys(saout,s)) temp_nomem();
@@ -299,6 +390,9 @@ void getcontrols()
  if (control_readint(&timeout,"control/timeoutremote") == -1) temp_control();
  if (control_readint(&timeoutconnect,"control/timeoutconnect") == -1)
  temp_control();
+#ifdef SMTPUTF8
+ if (control_readint(&enable_smtputf8,"control/smtputf8") == -1) temp_control();
+#endif
  if (control_rldef(&helohost,"control/helohost",1,NULL) != 1)
  temp_control();
  switch(control_readfile(&routes,"control/smtproutes",0)) {
@@ -325,7 +419,6 @@ int main(int argc, char **argv)
  if (chdir(auto_qmail) == -1) temp_chdir();
  getcontrols();
 
-
  if (!stralloc_copys(&host,argv[1])) temp_nomem();
 
  relayhost = 0;
@@ -343,8 +436,21 @@ int main(int argc, char **argv)
  }
  if (!stralloc_copys(&host,relayhost)) temp_nomem();
  }
+#ifdef SMTPUTF8
+ else
+ if (enable_smtputf8) {
+ char *asciihost = NULL;
+ if (!stralloc_0(&host)) temp_nomem();
+ switch (idn2_lookup_u8(host.s,(uint8_t**)&asciihost,IDN2_NFC_INPUT)) {
+ case IDN2_OK: break;
+ case IDN2_MALLOC: temp_nomem();
+ default: perm_dns();
+ }
+ if (!stralloc_copys(&idnhost,asciihost)) temp_nomem();
+ }
+#endif
 
-
+ /*- addrmangle also sets flagutf8 */
  addrmangle(&sender,argv[2]);
 
  if (!saa_readyplus(&reciplist,0)) temp_nomem();
@@ -361,7 +467,8 @@ int main(int argc, char **argv)
 
 
  random = now() + (getpid() << 16);
- switch (relayhost ? dns_ip(&ip,&host) : dns_mxip(&ip,&host,random)) {
+ i = relayhost ? dns_ip(&ip,&host) : dns_mxip(&ip,enable_smtputf8 ? &idnhost : &host,random);
+ switch (i) {
  case DNS_MEM: temp_nomem();
  case DNS_SOFT: temp_dns();
  case DNS_HARD: perm_dns();

qmail-smtpd.8

@@ -14,6 +14,11 @@ must be supplied several environment variables;
 see
 .BR tcp-environ(5) .
 
+.B qmail-smtpd
+offers RFC 5336 SMTP Email Address Internationalization support and will advertize the
+capability in the EHLO greeting. Since qmail-smtpd is 8 bit clean, setting of SMTPUTF8 has no real
+consequences except for displaying this setting in the received headers as \fBUTF8SMTP\fR.
+
 .B qmail-smtpd
 is responsible for counting hops.
 It rejects any message with 100 or more 
@@ -35,6 +40,7 @@ see
 .B qmail-smtpd
 accepts messages that contain long lines or non-ASCII characters,
 even though such messages violate the SMTP protocol.
+
 .SH "CONTROL FILES"
 .TP 5
 .I badmailfrom
@@ -169,6 +175,9 @@ Number of seconds
 .B qmail-smtpd
 will wait for each new buffer of data from the remote SMTP client.
 Default: 1200.
+.TP 5
+.I smtputf8
+Non-zero value in this control file enables RFC6531 SMTP Extension for Internationalized Email. 
 .SH "SEE ALSO"
 tcp-env(1),
 tcp-environ(5),