Ye_Ol_Pi_Shack
/
CppCMS
2
0
Fork 0
Code Issues 0 Pull Requests 0 Releases 1 Wiki Activity
ChipMaster's trial hacks on C++CMS starting with v1.2.1. Not sure I'll follow on with the v2 since it looks to be breaking and mostly frivolous.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
397 Commits
2 Branches
13 MiB
 
 
 
 
 
 
Tree: 2562c73a32
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '2562c73a32'
${ noResults }
CppCMS/encoding.cpp

535 lines
15 KiB
Raw Blame History 

  1. #define CPPCMS_SOURCE

  2. #include "encoding_validators.h"

  3. #include "encoding.h"

  4. #include "utf_iterator.h"

  5. #include "cppcms_error.h"

  6. #include "localization.h"

  7. #include "config.h"

  8. #include <errno.h>

  9. #include <string>

  10. #include <stdexcept>

  11. #include <iostream>

  12. 

  13. #ifdef HAVE_ICONV

  14. #include <iconv.h>

  15. #else

  16. #include <unicode/ucnv.h> // For Win32

  17. #endif

  18. 

  19. 

  20. 

  21. namespace cppcms {

  22. namespace encoding {

  23. 

  24. namespace impl{ 

  25. #ifdef HAVE_ICONV

  26. 	typedef size_t (*posix_iconv_type)(iconv_t cd,char **, size_t *t,char **,size_t *);

  27. 	typedef size_t (*gnu_iconv_type)(iconv_t cd,char const **, size_t *t,char **,size_t *);

  28. 

  29. 	size_t do_iconv(posix_iconv_type cv,iconv_t cd,char const **inbuf, size_t *inbytesleft,char **outbuf,size_t *outbytesleft)

  30. 	{

  31. 		return cv(cd,const_cast<char **>(inbuf),inbytesleft,outbuf,outbytesleft);

  32. 	}

  33. 	size_t do_iconv(gnu_iconv_type cv,iconv_t cd,char const **inbuf, size_t *inbytesleft,char **outbuf,size_t *outbytesleft)

  34. 	{

  35. 		return cv(cd,inbuf,inbytesleft,outbuf,outbytesleft);

  36. 	}

  37. 

  38. 	char const *native_utf32_encoding()

  39. 	{

  40. 		char const *le="UTF-32LE";

  41. 		char const *be="UTF-32BE";

  42. 		uint16_t v=0x0a0b;

  43. 		char const *utf=*(char*)&v== 0x0a ? be : le;

  44. 		return utf;

  45. 	}

  46. 	

  47. 	char const *native_utf16_encoding()

  48. 	{

  49. 		char const *le="UTF-16LE";

  50. 		char const *be="UTF-16BE";

  51. 		uint16_t v=0x0a0b;

  52. 		char const *utf=*(char*)&v== 0x0a ? be : le;

  53. 		return utf;

  54. 	}

  55. 

  56. 	char const *native_wchar_encoding()

  57. 	{

  58. 		if(sizeof(wchar_t)==4)

  59. 			return native_utf32_encoding();

  60. 		if(sizeof(wchar_t)==2)

  61. 			return native_utf16_encoding();

  62. 		throw std::runtime_error("wchar_t does not support unicode!");

  63. 	}

  64. 	

  65. 	class iconv_validator {

  66. 	public:

  67. 		iconv_validator(std::string const &charset) :

  68. 			descriptor_((iconv_t)(-1))

  69. 		{

  70. 			descriptor_=iconv_open(native_utf32_encoding(),charset.c_str());

  71. 			if(descriptor_==(iconv_t)(-1)) {

  72. 				throw std::runtime_error("Failed to load iconv tables for:" + charset);

  73. 			}

  74. 		}

  75. 

  76. 		~iconv_validator()

  77. 		{

  78. 			if(descriptor_!=(iconv_t)(-1))

  79. 				iconv_close(descriptor_);

  80. 		}

  81. 

  82. 		bool valid(char const *begin,char const *end,size_t &count)

  83. 		{

  84. 			iconv(descriptor_,0,0,0,0); // reset

  85. 			uint32_t buffer[64];

  86. 			size_t input=end-begin;

  87. 			while(begin!=end) {

  88. 				uint32_t *output=buffer;

  89. 				size_t outsize=sizeof(buffer);

  90. 

  91. 				size_t res=do_iconv(::iconv,descriptor_,&begin,&input,(char**)&output,&outsize);

  92. 

  93. 				if(res==(size_t)(-1) && errno==E2BIG)  {

  94. 					if(!check_symbols(buffer,output,count))

  95. 						return false;

  96. 					continue;

  97. 				}

  98. 				if(res!=(size_t)(-1) && input==0)

  99. 					return check_symbols(buffer,output,count);

  100. 				return false;

  101. 			}

  102. 			return true;

  103. 		}

  104. 	private:

  105. 		iconv_t descriptor_;

  106. 

  107. 		iconv_validator(iconv_validator const &other);

  108. 		iconv_validator const &operator=(iconv_validator const &);

  109. 

  110. 		bool check_symbols(uint32_t const *begin,uint32_t const *end,size_t &count)

  111. 		{

  112. 			while(begin!=end) {

  113. 				uint32_t c=*begin++;

  114. 				count++;

  115. 				if(c==0x09 || c==0xA || c==0xD)

  116. 					continue;

  117. 				if(c<0x20 || (0x7F<=c && c<=0x9F))

  118. 					return false;

  119. 			}

  120. 			return true;

  121. 		}

  122. 	};

  123. 

  124. 

  125. 	typedef iconv_validator validator;

  126. #else // NO HAVE_ICONV

  127. 	class uconv_validator {

  128. 	public:

  129. 		uconv_validator(std::string const &charset) :

  130. 			uconv_(0)

  131. 		{

  132. 			UErrorCode err=U_ZERO_ERROR;

  133. 			uconv_=ucnv_open(charset.c_str(),&err);

  134. 			if(!uconv_)

  135. 				throw cppcms_error("Invalid encoding:" + charset + u_errorName(err));

  136. 			err=U_ZERO_ERROR;

  137. 			ucnv_setToUCallBack(uconv_,UCNV_TO_U_CALLBACK_STOP,0,0,0,&err);

  138. 			if(U_FAILURE(err)) {

  139. 				ucnv_close(uconv_);

  140. 				throw cppcms_error("Invalid encoding:" + charset + u_errorName(err));

  141. 			}

  142. 		}

  143. 

  144. 		~uconv_validator()

  145. 		{

  146. 			if(uconv_) ucnv_close(uconv_);

  147. 		}

  148. 

  149. 		bool valid(char const *begin,char const *end,size_t &count)

  150. 		{

  151. 			UChar buffer[64];

  152. 			UChar *ubegin=buffer;

  153. 			UChar *uend=ubegin+64;

  154. 			count = 0;

  155. 			while(begin!=end) {

  156. 				UErrorCode err=U_ZERO_ERROR;

  157. 				ucnv_toUnicode(uconv_,&ubegin,uend,&begin,end,0,1,&err);

  158. 				if(err==U_BUFFER_OVERFLOW_ERROR) {

  159. 					if(!check_symbols(buffer,ubegin,count))

  160. 						return false;

  161. 				}

  162. 				else if(U_FAILURE(err))

  163. 					return false;

  164. 			}

  165. 			return true;

  166. 		}

  167. 

  168. 	private:

  169. 		UConverter *uconv_;

  170. 

  171. 		uconv_validator(uconv_validator const &other);

  172. 		void operator=(uconv_validator const &);

  173. 		

  174. 		bool check_symbols(UChar const *begin,UChar const *end,size_t &count)

  175. 		{

  176. 			while(begin!=end) {

  177. 				UChar c=*begin++;

  178. 				if(!U_IS_SURROGATE(c) || U_IS_SURROGATE_LEAD(c))

  179. 					count++;

  180. 				if(c==0x09 || c==0xA || c==0xD)

  181. 					continue;

  182. 				if(c<0x20 || (0x7F<=c && c<=0x9F))

  183. 					return false;

  184. 			}

  185. 			return true;

  186. 		}

  187. 	};

  188. 

  189. 	typedef uconv_validator validator;

  190. #endif

  191. 	struct validators_set {

  192. 		

  193. 		typedef bool (*encoding_tester_type)(char const *begin,char const *end,size_t &count);

  194. 

  195. 		encoding_tester_type get(char const *str) const

  196. 		{

  197. 			std::string name=str;

  198. 			for(unsigned i=0;i<name.size();i++)

  199. 				if('A' <= name[i]  && name[i] <= 'Z')

  200. 					name[i]-=('A'-'a');

  201. 			std::map<std::string,encoding_tester_type>::const_iterator p;

  202. 			p=predefined_.find(name);

  203. 			if(p==predefined_.end())

  204. 				return 0;

  205. 			return p->second;			

  206. 		}

  207. 

  208. 		validators_set() 

  209. 		{

  210. 

  211. 			encoding_tester_type iso_tester=&iso_8859_1_2_4_5_9_10_13_14_15_16_valid<char const *>;

  212. 

  213. 			

  214. 			predefined_["latin1"]=iso_tester;

  215. 			

  216. 			predefined_["iso88591"]=iso_tester;

  217. 			predefined_["iso88592"]=iso_tester;

  218. 			predefined_["iso88594"]=iso_tester;

  219. 			predefined_["iso88595"]=iso_tester;

  220. 			predefined_["iso88599"]=iso_tester;

  221. 			predefined_["iso885910"]=iso_tester;

  222. 			predefined_["iso885913"]=iso_tester;

  223. 			predefined_["iso885914"]=iso_tester;

  224. 			predefined_["iso885915"]=iso_tester;

  225. 			predefined_["iso885916"]=iso_tester;

  226. 

  227. 			predefined_["8859_1"]=iso_tester;

  228. 			predefined_["8859_2"]=iso_tester;

  229. 			predefined_["8859_4"]=iso_tester;

  230. 			predefined_["8859_5"]=iso_tester;

  231. 			predefined_["8859_9"]=iso_tester;

  232. 			predefined_["8859_10"]=iso_tester;

  233. 			predefined_["8859_13"]=iso_tester;

  234. 			predefined_["8859_14"]=iso_tester;

  235. 			predefined_["8859_15"]=iso_tester;

  236. 			predefined_["8859_16"]=iso_tester;

  237. 

  238. 			predefined_["iso8859-1"]=iso_tester;

  239. 			predefined_["iso8859-2"]=iso_tester;

  240. 			predefined_["iso8859-4"]=iso_tester;

  241. 			predefined_["iso8859-5"]=iso_tester;

  242. 			predefined_["iso8859-9"]=iso_tester;

  243. 			predefined_["iso8859-10"]=iso_tester;

  244. 			predefined_["iso8859-13"]=iso_tester;

  245. 			predefined_["iso8859-14"]=iso_tester;

  246. 			predefined_["iso8859-15"]=iso_tester;

  247. 			predefined_["iso8859-16"]=iso_tester;

  248. 			

  249. 			predefined_["iso_8859-1"]=iso_tester;

  250. 			predefined_["iso_8859-2"]=iso_tester;

  251. 			predefined_["iso_8859-4"]=iso_tester;

  252. 			predefined_["iso_8859-5"]=iso_tester;

  253. 			predefined_["iso_8859-9"]=iso_tester;

  254. 			predefined_["iso_8859-10"]=iso_tester;

  255. 			predefined_["iso_8859-13"]=iso_tester;

  256. 			predefined_["iso_8859-14"]=iso_tester;

  257. 			predefined_["iso_8859-15"]=iso_tester;

  258. 			predefined_["iso_8859-16"]=iso_tester;

  259. 

  260. 			predefined_["iso-8859-1"]=iso_tester;

  261. 			predefined_["iso-8859-2"]=iso_tester;

  262. 			predefined_["iso-8859-4"]=iso_tester;

  263. 			predefined_["iso-8859-5"]=iso_tester;

  264. 			predefined_["iso-8859-9"]=iso_tester;

  265. 			predefined_["iso-8859-10"]=iso_tester;

  266. 			predefined_["iso-8859-13"]=iso_tester;

  267. 			predefined_["iso-8859-14"]=iso_tester;

  268. 			predefined_["iso-8859-15"]=iso_tester;

  269. 			predefined_["iso-8859-16"]=iso_tester;

  270. 

  271. 			predefined_["iso88593"]=&iso_8859_3_valid<char const *>;

  272. 			predefined_["iso88596"]=&iso_8859_6_valid<char const *>;

  273. 			predefined_["iso88597"]=&iso_8859_7_valid<char const *>;

  274. 			predefined_["iso88598"]=&iso_8859_8_valid<char const *>;

  275. 			predefined_["iso885911"]=&iso_8859_11_valid<char const *>;

  276. 

  277. 			predefined_["iso8859-3"]=&iso_8859_3_valid<char const *>;

  278. 			predefined_["iso8859-6"]=&iso_8859_6_valid<char const *>;

  279. 			predefined_["iso8859-7"]=&iso_8859_7_valid<char const *>;

  280. 			predefined_["iso8859-8"]=&iso_8859_8_valid<char const *>;

  281. 			predefined_["iso8859-11"]=&iso_8859_11_valid<char const *>;

  282. 			

  283. 			predefined_["8859_3"]=&iso_8859_3_valid<char const *>;

  284. 			predefined_["8859_6"]=&iso_8859_6_valid<char const *>;

  285. 			predefined_["8859_7"]=&iso_8859_7_valid<char const *>;

  286. 			predefined_["8859_8"]=&iso_8859_8_valid<char const *>;

  287. 			predefined_["8859_11"]=&iso_8859_11_valid<char const *>;

  288. 

  289. 			predefined_["iso_8859-3"]=&iso_8859_3_valid<char const *>;

  290. 			predefined_["iso_8859-6"]=&iso_8859_6_valid<char const *>;

  291. 			predefined_["iso_8859-7"]=&iso_8859_7_valid<char const *>;

  292. 			predefined_["iso_8859-8"]=&iso_8859_8_valid<char const *>;

  293. 			predefined_["iso_8859-11"]=&iso_8859_11_valid<char const *>;

  294. 		

  295. 			predefined_["iso-8859-3"]=&iso_8859_3_valid<char const *>;

  296. 			predefined_["iso-8859-6"]=&iso_8859_6_valid<char const *>;

  297. 			predefined_["iso-8859-7"]=&iso_8859_7_valid<char const *>;

  298. 			predefined_["iso-8859-8"]=&iso_8859_8_valid<char const *>;

  299. 			predefined_["iso-8859-11"]=&iso_8859_11_valid<char const *>;

  300. 

  301. 			predefined_["windows-1250"]=&windows_1250_valid<char const *>;

  302. 			predefined_["windows-1251"]=&windows_1251_valid<char const *>;

  303. 			predefined_["windows-1252"]=&windows_1252_valid<char const *>;

  304. 			predefined_["windows-1253"]=&windows_1253_valid<char const *>;

  305. 			predefined_["windows-1255"]=&windows_1255_valid<char const *>;

  306. 			predefined_["windows-1256"]=&windows_1256_valid<char const *>;

  307. 			predefined_["windows-1257"]=&windows_1257_valid<char const *>;

  308. 			predefined_["windows-1258"]=&windows_1258_valid<char const *>;

  309. 

  310. 			predefined_["cp1250"]=&windows_1250_valid<char const *>;

  311. 			predefined_["cp1251"]=&windows_1251_valid<char const *>;

  312. 			predefined_["cp1252"]=&windows_1252_valid<char const *>;

  313. 			predefined_["cp1253"]=&windows_1253_valid<char const *>;

  314. 			predefined_["cp1255"]=&windows_1255_valid<char const *>;

  315. 			predefined_["cp1256"]=&windows_1256_valid<char const *>;

  316. 			predefined_["cp1257"]=&windows_1257_valid<char const *>;

  317. 			predefined_["cp1258"]=&windows_1258_valid<char const *>;

  318. 

  319. 			predefined_["1250"]=&windows_1250_valid<char const *>;

  320. 			predefined_["1251"]=&windows_1251_valid<char const *>;

  321. 			predefined_["1252"]=&windows_1252_valid<char const *>;

  322. 			predefined_["1253"]=&windows_1253_valid<char const *>;

  323. 			predefined_["1255"]=&windows_1255_valid<char const *>;

  324. 			predefined_["1256"]=&windows_1256_valid<char const *>;

  325. 			predefined_["1257"]=&windows_1257_valid<char const *>;

  326. 			predefined_["1258"]=&windows_1258_valid<char const *>;

  327. 

  328. 			predefined_["koi8r"]=predefined_["koi8-r"]=&koi8_valid<char const *>;

  329. 			predefined_["koi8u"]=predefined_["koi8-u"]=&koi8_valid<char const *>;

  330. 			

  331. 			predefined_["utf8"]=predefined_["utf-8"]=&utf8_valid<char const *>;

  332. 			predefined_["us-ascii"]=predefined_["ascii"]=&ascii_valid<char const *>;

  333. 		}

  334. 	private:

  335. 		std::map<std::string,encoding_tester_type> predefined_;

  336. 	} all_validators;

  337. } // impl

  338. 

  339. bool CPPCMS_API valid_utf8(char const *begin,char const *end,size_t &count)

  340. {

  341. 	return utf8_valid(begin,end,count);

  342. }

  343. 

  344. bool CPPCMS_API valid(std::string const &encoding,char const *begin,char const *end,size_t &count)

  345. {

  346. 	return valid(encoding.c_str(),begin,end,count);

  347. }

  348. 

  349. bool CPPCMS_API valid(std::locale const &loc,char const *begin,char const *end,size_t &count)

  350. {

  351. 	return valid(std::use_facet<locale::info>(loc).encoding().c_str(),begin,end,count);

  352. }

  353. 

  354. bool CPPCMS_API valid(char const *encoding,char const *begin,char const *end,size_t &count)

  355. {

  356. 	impl::validators_set::encoding_tester_type tester = impl::all_validators.get(encoding);

  357. 	if(tester)

  358. 		return tester(begin,end,count);

  359. 	try {

  360. 		impl::validator vtester(encoding);

  361. 		return vtester.valid(begin,end,count);

  362. 	}

  363. 	catch(std::runtime_error const &e) {

  364. 		return false;

  365. 	}

  366. }

  367. 

  368. 

  369. inline bool is_utf8(char const *c_encoding)

  370. {

  371. 	return 	strcmp(c_encoding,"UTF8")==0 

  372. 		|| strcmp(c_encoding,"UTF-8")==0

  373. 		|| strcmp(c_encoding,"utf8")==0

  374. 		|| strcmp(c_encoding,"utf-8")==0;

  375. }

  376. 

  377. 

  378. #ifdef HAVE_ICONV

  379. namespace impl {

  380. 	std::string iconv_convert_to(char const *to,char const *from,char const *begin,char const *end)

  381. 	{

  382. 		iconv_t d=iconv_open(to,from);

  383. 		if(d==(iconv_t)(-1)) 

  384. 			throw cppcms_error("Unsupported encoding "+std::string(to));

  385. 		

  386. 		std::string result;

  387. 		try {

  388. 			char buffer[256];

  389. 			size_t input=end-begin;

  390. 			while(begin!=end) {

  391. 				char *output=buffer;

  392. 				size_t outsize=sizeof(buffer);

  393. 

  394. 				size_t res=do_iconv(::iconv,d,&begin,&input,(char**)&output,&outsize);

  395. 

  396. 				if(res==(size_t)(-1) && errno==E2BIG)  {

  397. 					result.append(buffer,output);

  398. 					continue;

  399. 				}

  400. 				if(res!=(size_t)(-1) && input==0) {

  401. 					result.append(buffer,output);

  402. 					break;

  403. 				}

  404. 				break;

  405. 			}

  406. 			

  407. 		}

  408. 		catch(...) {

  409. 			iconv_close(d);

  410. 		}

  411. 		iconv_close(d);

  412. 		return result;

  413. 	}

  414. } // impl

  415. #endif

  416. 

  417. 

  418. std::string CPPCMS_API to_utf8(char const *c_encoding,char const *begin,char const *end)

  419. {

  420. 	std::string result;

  421. 	if(is_utf8(c_encoding)) {

  422. 		result.assign(begin,end-begin);

  423. 		return result;

  424. 	}

  425. #ifdef HAVE_ICONV

  426. 	return impl::iconv_convert_to("UTF-8",c_encoding,begin,end);

  427. #else // USE ICU

  428. 	locale::details::converter cvt(c_encoding);

  429. 	result.reserve(end-begin);

  430. 

  431. 	std::vector<char> buf(cvt.max_len() * 64);

  432. 

  433. 	while(begin<end) {

  434. 		uint32_t u=utf8::next(begin,end,false,true);

  435. 		if(u > 0x10FFFF) // error

  436. 			return result;

  437. 		char *tbegin=&buf[0];

  438. 		char *tend=tbegin+buf.size();

  439. 		uint32_t n = cvt.from_unicode(u ,tbegin,tend);

  440. 		if(n != locale::details::converter::illegal && n!= locale::details::converter::incomplete)

  441. 			result.append(tbegin,tbegin+n);

  442. 		else

  443. 			return result;

  444. 	}

  445. 	return result;

  446. #endif

  447. }

  448. 

  449. 

  450. std::string CPPCMS_API to_utf8(char const *encoding,std::string const &str)

  451. {

  452. 	if(is_utf8(encoding))

  453. 		return str;

  454. 	return to_utf8(encoding,str.data(),str.data()+str.size());

  455. }

  456. 

  457. std::string CPPCMS_API to_utf8(std::locale const &loc,char const *begin,char const *end)

  458. {

  459. 	locale::info const &inf = std::use_facet<locale::info>(loc);

  460. 	if(inf.utf8())

  461. 		return std::string(begin,end-begin);

  462. 	else

  463. 		return to_utf8(inf.encoding().c_str(),begin,end);

  464. }

  465. 

  466. std::string CPPCMS_API to_utf8(std::locale const &loc,std::string const &str)

  467. {

  468. 	locale::info const &inf = std::use_facet<locale::info>(loc);

  469. 	if(inf.utf8())

  470. 		return str;

  471. 	else

  472. 		return to_utf8(inf.encoding().c_str(),str);

  473. }

  474. 

  475. ////////////////////

  476. // FROM

  477. ///////////////////

  478. 

  479. 

  480. std::string CPPCMS_API from_utf8(char const *c_encoding,char const *begin,char const *end)

  481. {

  482. 	std::string result;

  483. 	if(is_utf8(c_encoding)) {

  484. 		result.assign(begin,end-begin);

  485. 		return result;

  486. 	}

  487. #ifdef HAVE_ICONV

  488. 	return impl::iconv_convert_to(c_encoding,"UTF-8",begin,end);

  489. #else // USE ICU

  490. 	locale::details::converter cvt(c_encoding);

  491. 	result.reserve(end-begin);

  492. 

  493. 	while(begin<end) {

  494. 		uint32_t u=cvt.to_unicode(begin,end);

  495. 		if(u > 0x10FFFF) // error

  496. 			return result;

  497. 		utf8::seq s=utf8::encode(u);

  498. 		result.append(s.c,s.len);

  499. 	}

  500. 	return result;

  501. #endif

  502. }

  503. 

  504. 

  505. 

  506. 

  507. std::string CPPCMS_API from_utf8(char const *encoding,std::string const &str)

  508. {

  509. 	if(is_utf8(encoding))

  510. 		return str;

  511. 	return from_utf8(encoding,str.data(),str.data()+str.size());

  512. }

  513. 

  514. std::string CPPCMS_API from_utf8(std::locale const &loc,char const *begin,char const *end)

  515. {

  516. 	locale::info const &inf = std::use_facet<locale::info>(loc);

  517. 	if(inf.utf8())

  518. 		return std::string(begin,end-begin);

  519. 	else

  520. 		return from_utf8(inf.encoding().c_str(),begin,end);

  521. }

  522. 

  523. std::string CPPCMS_API from_utf8(std::locale const &loc,std::string const &str)

  524. {

  525. 	locale::info const &inf = std::use_facet<locale::info>(loc);

  526. 	if(inf.utf8())

  527. 		return str;

  528. 	else

  529. 		return from_utf8(inf.encoding().c_str(),str);

  530. }

  531. 

  532. 	

  533. 	

  534. } } // cppcms::encoding