Ruby 4.0.10p0 (2026-11-19 revision a704b7cd95a42cc0cf5f307137bf3aac0285b915)
transcode.c
1/**********************************************************************
2
3 transcode.c -
4
5 $Author$
6 created at: Tue Oct 30 16:10:22 JST 2007
7
8 Copyright (C) 2007 Martin Duerst
9
10**********************************************************************/
11
12#include "ruby/internal/config.h"
13
14#include <ctype.h>
15
16#include "internal.h"
17#include "internal/array.h"
18#include "internal/inits.h"
19#include "internal/object.h"
20#include "internal/string.h"
21#include "internal/transcode.h"
22#include "internal/encoding.h"
23#include "ruby/encoding.h"
24#include "vm_sync.h"
25
26#include "transcode_data.h"
27#include "id.h"
28
29#define ENABLE_ECONV_NEWLINE_OPTION 1
30
31/* VALUE rb_cEncoding = rb_define_class("Encoding", rb_cObject); */
32static VALUE rb_eUndefinedConversionError;
33static VALUE rb_eInvalidByteSequenceError;
34static VALUE rb_eConverterNotFoundError;
35
36VALUE rb_cEncodingConverter;
37
38static ID id_destination_encoding;
39static ID id_destination_encoding_name;
40static ID id_error_bytes;
41static ID id_error_char;
42static ID id_incomplete_input;
43static ID id_readagain_bytes;
44static ID id_source_encoding;
45static ID id_source_encoding_name;
46
47static VALUE sym_invalid, sym_undef, sym_replace, sym_fallback;
48static VALUE sym_xml, sym_text, sym_attr;
49static VALUE sym_universal_newline;
50static VALUE sym_crlf_newline;
51static VALUE sym_cr_newline;
52static VALUE sym_lf_newline;
53#ifdef ENABLE_ECONV_NEWLINE_OPTION
54static VALUE sym_newline, sym_universal, sym_crlf, sym_cr, sym_lf;
55#endif
56static VALUE sym_partial_input;
57
58static VALUE sym_invalid_byte_sequence;
59static VALUE sym_undefined_conversion;
60static VALUE sym_destination_buffer_full;
61static VALUE sym_source_buffer_empty;
62static VALUE sym_finished;
63static VALUE sym_after_output;
64static VALUE sym_incomplete_input;
65
66static unsigned char *
67allocate_converted_string(const char *sname, const char *dname,
68 const unsigned char *str, size_t len,
69 unsigned char *caller_dst_buf, size_t caller_dst_bufsize,
70 size_t *dst_len_ptr);
71
72/* dynamic structure, one per conversion (similar to iconv_t) */
73/* may carry conversion state (e.g. for iso-2022-jp) */
74typedef struct rb_transcoding {
75 const rb_transcoder *transcoder;
76
77 int flags;
78
79 int resume_position;
80 unsigned int next_table;
81 VALUE next_info;
82 unsigned char next_byte;
83 unsigned int output_index;
84
85 ssize_t recognized_len; /* already interpreted */
86 ssize_t readagain_len; /* not yet interpreted */
87 union {
88 unsigned char ary[8]; /* max_input <= sizeof(ary) */
89 unsigned char *ptr; /* length: max_input */
90 } readbuf; /* recognized_len + readagain_len used */
91
92 ssize_t writebuf_off;
93 ssize_t writebuf_len;
94 union {
95 unsigned char ary[8]; /* max_output <= sizeof(ary) */
96 unsigned char *ptr; /* length: max_output */
97 } writebuf;
98
99 union rb_transcoding_state_t { /* opaque data for stateful encoding */
100 void *ptr;
101 char ary[sizeof(double) > sizeof(void*) ? sizeof(double) : sizeof(void*)];
102 double dummy_for_alignment;
103 } state;
105#define TRANSCODING_READBUF(tc) \
106 ((tc)->transcoder->max_input <= (int)sizeof((tc)->readbuf.ary) ? \
107 (tc)->readbuf.ary : \
108 (tc)->readbuf.ptr)
109#define TRANSCODING_WRITEBUF(tc) \
110 ((tc)->transcoder->max_output <= (int)sizeof((tc)->writebuf.ary) ? \
111 (tc)->writebuf.ary : \
112 (tc)->writebuf.ptr)
113#define TRANSCODING_WRITEBUF_SIZE(tc) \
114 ((tc)->transcoder->max_output <= (int)sizeof((tc)->writebuf.ary) ? \
115 sizeof((tc)->writebuf.ary) : \
116 (size_t)(tc)->transcoder->max_output)
117#define TRANSCODING_STATE_EMBED_MAX ((int)sizeof(union rb_transcoding_state_t))
118#define TRANSCODING_STATE(tc) \
119 ((tc)->transcoder->state_size <= (int)sizeof((tc)->state) ? \
120 (tc)->state.ary : \
121 (tc)->state.ptr)
122
123typedef struct {
124 struct rb_transcoding *tc;
125 unsigned char *out_buf_start;
126 unsigned char *out_data_start;
127 unsigned char *out_data_end;
128 unsigned char *out_buf_end;
129 rb_econv_result_t last_result;
131
133 int flags;
134 int started; /* bool */
135
136 const char *source_encoding_name;
137 const char *destination_encoding_name;
138
139 const unsigned char *replacement_str;
140 size_t replacement_len;
141 const char *replacement_enc;
142
143 unsigned char *in_buf_start;
144 unsigned char *in_data_start;
145 unsigned char *in_data_end;
146 unsigned char *in_buf_end;
147 rb_econv_elem_t *elems;
148 int replacement_allocated; /* bool */
149 int num_allocated;
150 int num_trans;
151 int num_finished;
152 struct rb_transcoding *last_tc;
153
154 /* last error */
155 struct {
156 rb_econv_result_t result;
157 struct rb_transcoding *error_tc;
158 const char *source_encoding;
159 const char *destination_encoding;
160 const unsigned char *error_bytes_start;
161 size_t error_bytes_len;
162 size_t readagain_len;
163 } last_error;
164
165 /* The following fields are only for Encoding::Converter.
166 * rb_econv_open set them NULL. */
167 rb_encoding *source_encoding;
168 rb_encoding *destination_encoding;
169};
170
171/*
172 * Dispatch data and logic
173 */
174
175#define DECORATOR_P(sname, dname) (*(sname) == '\0')
176
177typedef struct {
178 const char *sname;
179 const char *dname;
180 const char *lib; /* null means no need to load a library */
181 const rb_transcoder *transcoder;
183
184static st_table *transcoder_table;
185
186static int
187free_inner_transcode_i(st_data_t key, st_data_t val, st_data_t arg)
188{
189 xfree((void *)val);
190 return ST_DELETE;
191}
192
193static int
194free_transcode_i(st_data_t key, st_data_t val, st_data_t arg)
195{
196 st_foreach((void *)val, free_inner_transcode_i, 0);
197 st_free_table((void *)val);
198 return ST_DELETE;
199}
200
201void
202rb_free_transcoder_table(void)
203{
204 st_foreach(transcoder_table, free_transcode_i, 0);
205 st_free_table(transcoder_table);
206}
207
208static transcoder_entry_t *
209make_transcoder_entry(const char *sname, const char *dname)
210{
211 st_data_t val;
212 st_table *table2;
213
214 RB_VM_LOCKING() {
215 if (!st_lookup(transcoder_table, (st_data_t)sname, &val)) {
216 val = (st_data_t)st_init_strcasetable();
217 st_add_direct(transcoder_table, (st_data_t)sname, val);
218 }
219 table2 = (st_table *)val;
220 if (!st_lookup(table2, (st_data_t)dname, &val)) {
222 entry->sname = sname;
223 entry->dname = dname;
224 entry->lib = NULL;
225 entry->transcoder = NULL;
226 val = (st_data_t)entry;
227 st_add_direct(table2, (st_data_t)dname, val);
228 }
229 }
230 return (transcoder_entry_t *)val;
231}
232
233static transcoder_entry_t *
234get_transcoder_entry(const char *sname, const char *dname)
235{
236 st_data_t val = 0;
237 st_table *table2;
238 RB_VM_LOCKING() {
239 if (st_lookup(transcoder_table, (st_data_t)sname, &val)) {
240 table2 = (st_table *)val;
241 if (!st_lookup(table2, (st_data_t)dname, &val)) {
242 val = 0;
243 }
244 }
245 }
246 return (transcoder_entry_t *)val;
247}
248
249void
250rb_register_transcoder(const rb_transcoder *tr)
251{
252 const char *const sname = tr->src_encoding;
253 const char *const dname = tr->dst_encoding;
254
255 transcoder_entry_t *entry;
256
257 RB_VM_LOCKING() {
258 entry = make_transcoder_entry(sname, dname);
259 if (entry->transcoder) {
260 rb_raise(rb_eArgError, "transcoder from %s to %s has been already registered",
261 sname, dname);
262 }
263 entry->transcoder = tr;
264 }
265}
266
267static void
268declare_transcoder(const char *sname, const char *dname, const char *lib)
269{
270 transcoder_entry_t *entry;
271
272 entry = make_transcoder_entry(sname, dname);
273 entry->lib = lib;
274}
275
276static const char transcoder_lib_prefix[] = "enc/trans/";
277
278void
279rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib)
280{
281 if (!lib) {
282 rb_raise(rb_eArgError, "invalid library name - (null)");
283 }
284 declare_transcoder(enc1, enc2, lib);
285}
286
287#define encoding_equal(enc1, enc2) (STRCASECMP((enc1), (enc2)) == 0)
288
289typedef struct search_path_queue_tag {
290 struct search_path_queue_tag *next;
291 const char *enc;
292} search_path_queue_t;
293
294typedef struct {
295 st_table *visited;
296 search_path_queue_t *queue;
297 search_path_queue_t **queue_last_ptr;
298 const char *base_enc;
300
301static int
302transcode_search_path_i(st_data_t key, st_data_t val, st_data_t arg)
303{
304 const char *dname = (const char *)key;
306 search_path_queue_t *q;
307
308 if (st_lookup(bfs->visited, (st_data_t)dname, &val)) {
309 return ST_CONTINUE;
310 }
311
312 q = ALLOC(search_path_queue_t);
313 q->enc = dname;
314 q->next = NULL;
315 *bfs->queue_last_ptr = q;
316 bfs->queue_last_ptr = &q->next;
317
318 st_add_direct(bfs->visited, (st_data_t)dname, (st_data_t)bfs->base_enc);
319 return ST_CONTINUE;
320}
321
322static int
323transcode_search_path(const char *sname, const char *dname,
324 void (*callback)(const char *sname, const char *dname, int depth, void *arg),
325 void *arg)
326{
328 search_path_queue_t *q;
329 st_data_t val;
330 st_table *table2;
331 int pathlen = -1;
332 bool found = false;
333 bool lookup_res;
334
335 if (encoding_equal(sname, dname))
336 return -1;
337
338 q = ALLOC(search_path_queue_t);
339 q->enc = sname;
340 q->next = NULL;
341 bfs.queue_last_ptr = &q->next;
342 bfs.queue = q;
343
344 bfs.visited = st_init_strcasetable(); // due to base encodings, we need to do search in a loop
345 st_add_direct(bfs.visited, (st_data_t)sname, (st_data_t)NULL);
346
347 RB_VM_LOCKING() {
348 while (bfs.queue) {
349 q = bfs.queue;
350 bfs.queue = q->next;
351 if (!bfs.queue) {
352 bfs.queue_last_ptr = &bfs.queue;
353 }
354
355 lookup_res = st_lookup(transcoder_table, (st_data_t)q->enc, &val); // src => table2
356 if (!lookup_res) {
357 xfree(q);
358 continue;
359 }
360 table2 = (st_table *)val;
361
362 if (st_lookup(table2, (st_data_t)dname, &val)) { // dest => econv
363 st_add_direct(bfs.visited, (st_data_t)dname, (st_data_t)q->enc);
364 xfree(q);
365 found = true;
366 break;
367 }
368
369 bfs.base_enc = q->enc;
370 st_foreach(table2, transcode_search_path_i, (st_data_t)&bfs);
371
372 bfs.base_enc = NULL;
373 xfree(q);
374 }
375 }
376
377 while (bfs.queue) {
378 q = bfs.queue;
379 bfs.queue = q->next;
380 xfree(q);
381 }
382
383 if (found) {
384 const char *enc = dname;
385 int depth;
386 pathlen = 0;
387 while (1) {
388 st_lookup(bfs.visited, (st_data_t)enc, &val);
389 if (!val)
390 break;
391 pathlen++;
392 enc = (const char *)val;
393 }
394 depth = pathlen;
395 enc = dname;
396 while (1) {
397 st_lookup(bfs.visited, (st_data_t)enc, &val);
398 if (!val)
399 break;
400 callback((const char *)val, enc, --depth, arg);
401 enc = (const char *)val;
402 }
403 }
404
405 st_free_table(bfs.visited);
406
407 return pathlen; /* is -1 if not found */
408}
409
410int rb_require_internal_silent(VALUE fname);
411
412static const rb_transcoder *
413load_transcoder_entry(transcoder_entry_t *entry)
414{
415 ASSERT_vm_unlocking();
416 if (entry->transcoder)
417 return entry->transcoder;
418
419 if (entry->lib) {
420 const char *const lib = entry->lib;
421 const size_t len = strlen(lib);
422 const size_t total_len = sizeof(transcoder_lib_prefix) - 1 + len;
423 const VALUE fn = rb_str_new(0, total_len);
424 char *const path = RSTRING_PTR(fn);
425
426 memcpy(path, transcoder_lib_prefix, sizeof(transcoder_lib_prefix) - 1);
427 memcpy(path + sizeof(transcoder_lib_prefix) - 1, lib, len);
428 rb_str_set_len(fn, total_len);
429 OBJ_FREEZE(fn);
430 rb_require_internal_silent(fn); // Sets entry->transcoder
431 }
432
433 if (entry->transcoder)
434 return entry->transcoder;
435
436 return NULL;
437}
438
439static const char*
440get_replacement_character(const char *encname, size_t *len_ret, const char **repl_encname_ptr)
441{
442 if (encoding_equal(encname, "UTF-8")) {
443 *len_ret = 3;
444 *repl_encname_ptr = "UTF-8";
445 return "\xEF\xBF\xBD";
446 }
447 else {
448 *len_ret = 1;
449 *repl_encname_ptr = "US-ASCII";
450 return "?";
451 }
452}
453
454/*
455 * Transcoding engine logic
456 */
457
458static const unsigned char *
459transcode_char_start(rb_transcoding *tc,
460 const unsigned char *in_start,
461 const unsigned char *inchar_start,
462 const unsigned char *in_p,
463 size_t *char_len_ptr)
464{
465 const unsigned char *ptr;
466 if (inchar_start - in_start < tc->recognized_len) {
467 MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len,
468 inchar_start, unsigned char, in_p - inchar_start);
469 ptr = TRANSCODING_READBUF(tc);
470 }
471 else {
472 ptr = inchar_start - tc->recognized_len;
473 }
474 *char_len_ptr = tc->recognized_len + (in_p - inchar_start);
475 return ptr;
476}
477
479transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos,
480 const unsigned char *in_stop, unsigned char *out_stop,
481 rb_transcoding *tc,
482 const int opt)
483{
484 const rb_transcoder *tr = tc->transcoder;
485 int unitlen = tr->input_unit_length;
486 ssize_t readagain_len = 0;
487
488 const unsigned char *inchar_start;
489 const unsigned char *in_p;
490
491 unsigned char *out_p;
492
493 in_p = inchar_start = *in_pos;
494
495 out_p = *out_pos;
496
497#define SUSPEND(ret, num) \
498 do { \
499 tc->resume_position = (num); \
500 if (0 < in_p - inchar_start) \
501 MEMMOVE(TRANSCODING_READBUF(tc)+tc->recognized_len, \
502 inchar_start, unsigned char, in_p - inchar_start); \
503 *in_pos = in_p; \
504 *out_pos = out_p; \
505 tc->recognized_len += in_p - inchar_start; \
506 if (readagain_len) { \
507 tc->recognized_len -= readagain_len; \
508 tc->readagain_len = readagain_len; \
509 } \
510 return (ret); \
511 resume_label ## num:; \
512 } while (0)
513#define SUSPEND_OBUF(num) \
514 do { \
515 while (out_stop - out_p < 1) { SUSPEND(econv_destination_buffer_full, num); } \
516 } while (0)
517
518#define SUSPEND_AFTER_OUTPUT(num) \
519 if ((opt & ECONV_AFTER_OUTPUT) && *out_pos != out_p) { \
520 SUSPEND(econv_after_output, num); \
521 }
522
523#define next_table (tc->next_table)
524#define next_info (tc->next_info)
525#define next_byte (tc->next_byte)
526#define writebuf_len (tc->writebuf_len)
527#define writebuf_off (tc->writebuf_off)
528
529 switch (tc->resume_position) {
530 case 0: break;
531 case 1: goto resume_label1;
532 case 2: goto resume_label2;
533 case 3: goto resume_label3;
534 case 4: goto resume_label4;
535 case 5: goto resume_label5;
536 case 6: goto resume_label6;
537 case 7: goto resume_label7;
538 case 8: goto resume_label8;
539 case 9: goto resume_label9;
540 case 10: goto resume_label10;
541 case 11: goto resume_label11;
542 case 12: goto resume_label12;
543 case 13: goto resume_label13;
544 case 14: goto resume_label14;
545 case 15: goto resume_label15;
546 case 16: goto resume_label16;
547 case 17: goto resume_label17;
548 case 18: goto resume_label18;
549 case 19: goto resume_label19;
550 case 20: goto resume_label20;
551 case 21: goto resume_label21;
552 case 22: goto resume_label22;
553 case 23: goto resume_label23;
554 case 24: goto resume_label24;
555 case 25: goto resume_label25;
556 case 26: goto resume_label26;
557 case 27: goto resume_label27;
558 case 28: goto resume_label28;
559 case 29: goto resume_label29;
560 case 30: goto resume_label30;
561 case 31: goto resume_label31;
562 case 32: goto resume_label32;
563 case 33: goto resume_label33;
564 case 34: goto resume_label34;
565 }
566
567 while (1) {
568 inchar_start = in_p;
569 tc->recognized_len = 0;
570 next_table = tr->conv_tree_start;
571
572 SUSPEND_AFTER_OUTPUT(24);
573
574 if (in_stop <= in_p) {
575 if (!(opt & ECONV_PARTIAL_INPUT))
576 break;
577 SUSPEND(econv_source_buffer_empty, 7);
578 continue;
579 }
580
581#define BYTE_ADDR(index) (tr->byte_array + (index))
582#define WORD_ADDR(index) (tr->word_array + INFO2WORDINDEX(index))
583#define BL_BASE BYTE_ADDR(BYTE_LOOKUP_BASE(WORD_ADDR(next_table)))
584#define BL_INFO WORD_ADDR(BYTE_LOOKUP_INFO(WORD_ADDR(next_table)))
585#define BL_MIN_BYTE (BL_BASE[0])
586#define BL_MAX_BYTE (BL_BASE[1])
587#define BL_OFFSET(byte) (BL_BASE[2+(byte)-BL_MIN_BYTE])
588#define BL_ACTION(byte) (BL_INFO[BL_OFFSET((byte))])
589
590 next_byte = (unsigned char)*in_p++;
591 follow_byte:
592 if (next_byte < BL_MIN_BYTE || BL_MAX_BYTE < next_byte)
593 next_info = INVALID;
594 else {
595 next_info = (VALUE)BL_ACTION(next_byte);
596 }
597 follow_info:
598 switch (next_info & 0x1F) {
599 case NOMAP:
600 {
601 const unsigned char *p = inchar_start;
602 writebuf_off = 0;
603 while (p < in_p) {
604 TRANSCODING_WRITEBUF(tc)[writebuf_off++] = (unsigned char)*p++;
605 }
606 writebuf_len = writebuf_off;
607 writebuf_off = 0;
608 while (writebuf_off < writebuf_len) {
609 SUSPEND_OBUF(3);
610 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
611 }
612 }
613 continue;
614 case 0x00: case 0x04: case 0x08: case 0x0C:
615 case 0x10: case 0x14: case 0x18: case 0x1C:
616 SUSPEND_AFTER_OUTPUT(25);
617 while (in_p >= in_stop) {
618 if (!(opt & ECONV_PARTIAL_INPUT))
619 goto incomplete;
620 SUSPEND(econv_source_buffer_empty, 5);
621 }
622 next_byte = (unsigned char)*in_p++;
623 next_table = (unsigned int)next_info;
624 goto follow_byte;
625 case ZERObt: /* drop input */
626 continue;
627 case ONEbt:
628 SUSPEND_OBUF(9); *out_p++ = getBT1(next_info);
629 continue;
630 case TWObt:
631 SUSPEND_OBUF(10); *out_p++ = getBT1(next_info);
632 SUSPEND_OBUF(21); *out_p++ = getBT2(next_info);
633 continue;
634 case THREEbt:
635 SUSPEND_OBUF(11); *out_p++ = getBT1(next_info);
636 SUSPEND_OBUF(15); *out_p++ = getBT2(next_info);
637 SUSPEND_OBUF(16); *out_p++ = getBT3(next_info);
638 continue;
639 case FOURbt:
640 SUSPEND_OBUF(12); *out_p++ = getBT0(next_info);
641 SUSPEND_OBUF(17); *out_p++ = getBT1(next_info);
642 SUSPEND_OBUF(18); *out_p++ = getBT2(next_info);
643 SUSPEND_OBUF(19); *out_p++ = getBT3(next_info);
644 continue;
645 case GB4bt:
646 SUSPEND_OBUF(29); *out_p++ = getGB4bt0(next_info);
647 SUSPEND_OBUF(30); *out_p++ = getGB4bt1(next_info);
648 SUSPEND_OBUF(31); *out_p++ = getGB4bt2(next_info);
649 SUSPEND_OBUF(32); *out_p++ = getGB4bt3(next_info);
650 continue;
651 case STR1:
652 tc->output_index = 0;
653 while (tc->output_index < STR1_LENGTH(BYTE_ADDR(STR1_BYTEINDEX(next_info)))) {
654 SUSPEND_OBUF(28); *out_p++ = BYTE_ADDR(STR1_BYTEINDEX(next_info))[1+tc->output_index];
655 tc->output_index++;
656 }
657 continue;
658 case FUNii:
659 next_info = (VALUE)(*tr->func_ii)(TRANSCODING_STATE(tc), next_info);
660 goto follow_info;
661 case FUNsi:
662 {
663 const unsigned char *char_start;
664 size_t char_len;
665 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
666 next_info = (VALUE)(*tr->func_si)(TRANSCODING_STATE(tc), char_start, (size_t)char_len);
667 goto follow_info;
668 }
669 case FUNio:
670 SUSPEND_OBUF(13);
671 if (tr->max_output <= out_stop - out_p)
672 out_p += tr->func_io(TRANSCODING_STATE(tc),
673 next_info, out_p, out_stop - out_p);
674 else {
675 writebuf_len = tr->func_io(TRANSCODING_STATE(tc),
676 next_info,
677 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
678 writebuf_off = 0;
679 while (writebuf_off < writebuf_len) {
680 SUSPEND_OBUF(20);
681 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
682 }
683 }
684 break;
685 case FUNso:
686 {
687 const unsigned char *char_start;
688 size_t char_len;
689 SUSPEND_OBUF(14);
690 if (tr->max_output <= out_stop - out_p) {
691 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
692 out_p += tr->func_so(TRANSCODING_STATE(tc),
693 char_start, (size_t)char_len,
694 out_p, out_stop - out_p);
695 }
696 else {
697 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
698 writebuf_len = tr->func_so(TRANSCODING_STATE(tc),
699 char_start, (size_t)char_len,
700 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
701 writebuf_off = 0;
702 while (writebuf_off < writebuf_len) {
703 SUSPEND_OBUF(22);
704 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
705 }
706 }
707 break;
708 }
709 case FUNsio:
710 {
711 const unsigned char *char_start;
712 size_t char_len;
713 SUSPEND_OBUF(33);
714 if (tr->max_output <= out_stop - out_p) {
715 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
716 out_p += tr->func_sio(TRANSCODING_STATE(tc),
717 char_start, (size_t)char_len, next_info,
718 out_p, out_stop - out_p);
719 }
720 else {
721 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
722 writebuf_len = tr->func_sio(TRANSCODING_STATE(tc),
723 char_start, (size_t)char_len, next_info,
724 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
725 writebuf_off = 0;
726 while (writebuf_off < writebuf_len) {
727 SUSPEND_OBUF(34);
728 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
729 }
730 }
731 break;
732 }
733 case INVALID:
734 if (tc->recognized_len + (in_p - inchar_start) <= unitlen) {
735 if (tc->recognized_len + (in_p - inchar_start) < unitlen)
736 SUSPEND_AFTER_OUTPUT(26);
737 while ((opt & ECONV_PARTIAL_INPUT) && tc->recognized_len + (in_stop - inchar_start) < unitlen) {
738 in_p = in_stop;
739 SUSPEND(econv_source_buffer_empty, 8);
740 }
741 if (tc->recognized_len + (in_stop - inchar_start) <= unitlen) {
742 in_p = in_stop;
743 }
744 else {
745 in_p = inchar_start + (unitlen - tc->recognized_len);
746 }
747 }
748 else {
749 ssize_t invalid_len; /* including the last byte which causes invalid */
750 ssize_t discard_len;
751 invalid_len = tc->recognized_len + (in_p - inchar_start);
752 discard_len = ((invalid_len - 1) / unitlen) * unitlen;
753 readagain_len = invalid_len - discard_len;
754 }
755 goto invalid;
756 case UNDEF:
757 goto undef;
758 default:
759 rb_raise(rb_eRuntimeError, "unknown transcoding instruction");
760 }
761 continue;
762
763 invalid:
764 SUSPEND(econv_invalid_byte_sequence, 1);
765 continue;
766
767 incomplete:
768 SUSPEND(econv_incomplete_input, 27);
769 continue;
770
771 undef:
772 SUSPEND(econv_undefined_conversion, 2);
773 continue;
774 }
775
776 /* cleanup */
777 if (tr->finish_func) {
778 SUSPEND_OBUF(4);
779 if (tr->max_output <= out_stop - out_p) {
780 out_p += tr->finish_func(TRANSCODING_STATE(tc),
781 out_p, out_stop - out_p);
782 }
783 else {
784 writebuf_len = tr->finish_func(TRANSCODING_STATE(tc),
785 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
786 writebuf_off = 0;
787 while (writebuf_off < writebuf_len) {
788 SUSPEND_OBUF(23);
789 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
790 }
791 }
792 }
793 while (1)
794 SUSPEND(econv_finished, 6);
795#undef SUSPEND
796#undef next_table
797#undef next_info
798#undef next_byte
799#undef writebuf_len
800#undef writebuf_off
801}
802
804transcode_restartable(const unsigned char **in_pos, unsigned char **out_pos,
805 const unsigned char *in_stop, unsigned char *out_stop,
806 rb_transcoding *tc,
807 const int opt)
808{
809 if (tc->readagain_len) {
810 unsigned char *readagain_buf = ALLOCA_N(unsigned char, tc->readagain_len);
811 const unsigned char *readagain_pos = readagain_buf;
812 const unsigned char *readagain_stop = readagain_buf + tc->readagain_len;
814
815 MEMCPY(readagain_buf, TRANSCODING_READBUF(tc) + tc->recognized_len,
816 unsigned char, tc->readagain_len);
817 tc->readagain_len = 0;
818 res = transcode_restartable0(&readagain_pos, out_pos, readagain_stop, out_stop, tc, opt|ECONV_PARTIAL_INPUT);
819 if (res != econv_source_buffer_empty) {
820 MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len,
821 readagain_pos, unsigned char, readagain_stop - readagain_pos);
822 tc->readagain_len += readagain_stop - readagain_pos;
823 return res;
824 }
825 }
826 return transcode_restartable0(in_pos, out_pos, in_stop, out_stop, tc, opt);
827}
828
829static rb_transcoding *
830rb_transcoding_open_by_transcoder(const rb_transcoder *tr, int flags)
831{
832 rb_transcoding *tc;
833
834 tc = ALLOC(rb_transcoding);
835 tc->transcoder = tr;
836 tc->flags = flags;
837 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size)
838 tc->state.ptr = xmalloc(tr->state_size);
839 if (tr->state_init_func) {
840 (tr->state_init_func)(TRANSCODING_STATE(tc)); /* xxx: check return value */
841 }
842 tc->resume_position = 0;
843 tc->recognized_len = 0;
844 tc->readagain_len = 0;
845 tc->writebuf_len = 0;
846 tc->writebuf_off = 0;
847 if ((int)sizeof(tc->readbuf.ary) < tr->max_input) {
848 tc->readbuf.ptr = xmalloc(tr->max_input);
849 }
850 if ((int)sizeof(tc->writebuf.ary) < tr->max_output) {
851 tc->writebuf.ptr = xmalloc(tr->max_output);
852 }
853 return tc;
854}
855
857rb_transcoding_convert(rb_transcoding *tc,
858 const unsigned char **input_ptr, const unsigned char *input_stop,
859 unsigned char **output_ptr, unsigned char *output_stop,
860 int flags)
861{
862 return transcode_restartable(
863 input_ptr, output_ptr,
864 input_stop, output_stop,
865 tc, flags);
866}
867
868static void
869rb_transcoding_close(rb_transcoding *tc)
870{
871 const rb_transcoder *tr = tc->transcoder;
872 if (tr->state_fini_func) {
873 (tr->state_fini_func)(TRANSCODING_STATE(tc)); /* check return value? */
874 }
875 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size)
876 xfree(tc->state.ptr);
877 if ((int)sizeof(tc->readbuf.ary) < tr->max_input)
878 xfree(tc->readbuf.ptr);
879 if ((int)sizeof(tc->writebuf.ary) < tr->max_output)
880 xfree(tc->writebuf.ptr);
881 xfree(tc);
882}
883
884static size_t
885rb_transcoding_memsize(rb_transcoding *tc)
886{
887 size_t size = sizeof(rb_transcoding);
888 const rb_transcoder *tr = tc->transcoder;
889
890 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size) {
891 size += tr->state_size;
892 }
893 if ((int)sizeof(tc->readbuf.ary) < tr->max_input) {
894 size += tr->max_input;
895 }
896 if ((int)sizeof(tc->writebuf.ary) < tr->max_output) {
897 size += tr->max_output;
898 }
899 return size;
900}
901
902static rb_econv_t *
903rb_econv_alloc(int n_hint)
904{
905 rb_econv_t *ec;
906
907 if (n_hint <= 0)
908 n_hint = 1;
909
910 ec = ALLOC(rb_econv_t);
911 ec->flags = 0;
912 ec->source_encoding_name = NULL;
913 ec->destination_encoding_name = NULL;
914 ec->started = 0;
915 ec->replacement_str = NULL;
916 ec->replacement_len = 0;
917 ec->replacement_enc = NULL;
918 ec->replacement_allocated = 0;
919 ec->in_buf_start = NULL;
920 ec->in_data_start = NULL;
921 ec->in_data_end = NULL;
922 ec->in_buf_end = NULL;
923 ec->num_allocated = n_hint;
924 ec->num_trans = 0;
925 ec->elems = ALLOC_N(rb_econv_elem_t, ec->num_allocated);
926 ec->num_finished = 0;
927 ec->last_tc = NULL;
928 ec->last_error.result = econv_source_buffer_empty;
929 ec->last_error.error_tc = NULL;
930 ec->last_error.source_encoding = NULL;
931 ec->last_error.destination_encoding = NULL;
932 ec->last_error.error_bytes_start = NULL;
933 ec->last_error.error_bytes_len = 0;
934 ec->last_error.readagain_len = 0;
935 ec->source_encoding = NULL;
936 ec->destination_encoding = NULL;
937 return ec;
938}
939
940static int
941rb_econv_add_transcoder_at(rb_econv_t *ec, const rb_transcoder *tr, int i)
942{
943 int n, j;
944 int bufsize = 4096;
945 unsigned char *p;
946
947 if (ec->num_trans == ec->num_allocated) {
948 n = ec->num_allocated * 2;
949 REALLOC_N(ec->elems, rb_econv_elem_t, n);
950 ec->num_allocated = n;
951 }
952
953 p = xmalloc(bufsize);
954
955 MEMMOVE(ec->elems+i+1, ec->elems+i, rb_econv_elem_t, ec->num_trans-i);
956
957 ec->elems[i].tc = rb_transcoding_open_by_transcoder(tr, 0);
958 ec->elems[i].out_buf_start = p;
959 ec->elems[i].out_buf_end = p + bufsize;
960 ec->elems[i].out_data_start = p;
961 ec->elems[i].out_data_end = p;
962 ec->elems[i].last_result = econv_source_buffer_empty;
963
964 ec->num_trans++;
965
966 if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding))
967 for (j = ec->num_trans-1; i <= j; j--) {
968 rb_transcoding *tc = ec->elems[j].tc;
969 const rb_transcoder *tr2 = tc->transcoder;
970 if (!DECORATOR_P(tr2->src_encoding, tr2->dst_encoding)) {
971 ec->last_tc = tc;
972 break;
973 }
974 }
975
976 return 0;
977}
978
979static rb_econv_t *
980rb_econv_open_by_transcoder_entries(int n, transcoder_entry_t **entries)
981{
982 rb_econv_t *ec;
983 int i, ret;
984
985 for (i = 0; i < n; i++) {
986 const rb_transcoder *tr;
987 tr = load_transcoder_entry(entries[i]);
988 if (!tr)
989 return NULL;
990 }
991
992 ec = rb_econv_alloc(n);
993
994 for (i = 0; i < n; i++) {
995 const rb_transcoder *tr = load_transcoder_entry(entries[i]);
996 ret = rb_econv_add_transcoder_at(ec, tr, ec->num_trans);
997 if (ret == -1) {
998 rb_econv_close(ec);
999 return NULL;
1000 }
1001 }
1002
1003 return ec;
1004}
1005
1007 transcoder_entry_t **entries;
1008 int num_additional;
1009};
1010
1011static void
1012trans_open_i(const char *sname, const char *dname, int depth, void *arg)
1013{
1014 struct trans_open_t *toarg = arg;
1015
1016 if (!toarg->entries) {
1017 toarg->entries = ALLOC_N(transcoder_entry_t *, depth+1+toarg->num_additional);
1018 }
1019 toarg->entries[depth] = get_transcoder_entry(sname, dname);
1020}
1021
1022static rb_econv_t *
1023rb_econv_open0(const char *sname, const char *dname, int ecflags)
1024{
1025 transcoder_entry_t **entries = NULL;
1026 int num_trans;
1027 rb_econv_t *ec;
1028
1029 // loads encodings if not loaded already
1030 if (*sname) rb_enc_find_index(sname);
1031 if (*dname) rb_enc_find_index(dname);
1032
1033 if (*sname == '\0' && *dname == '\0') {
1034 num_trans = 0;
1035 entries = NULL;
1036 sname = dname = "";
1037 }
1038 else {
1039 struct trans_open_t toarg;
1040 toarg.entries = NULL;
1041 toarg.num_additional = 0;
1042 num_trans = transcode_search_path(sname, dname, trans_open_i, (void *)&toarg);
1043 entries = toarg.entries;
1044 if (num_trans < 0) {
1045 xfree(entries);
1046 return NULL;
1047 }
1048 sname = entries[0]->sname;
1049 dname = entries[num_trans-1]->dname;
1050 }
1051
1052 ec = rb_econv_open_by_transcoder_entries(num_trans, entries);
1053 xfree(entries);
1054 if (!ec)
1055 return NULL;
1056
1057 ec->flags = ecflags;
1058 ec->source_encoding_name = sname;
1059 ec->destination_encoding_name = dname;
1060
1061 return ec;
1062}
1063
1064#define MAX_ECFLAGS_DECORATORS 32
1065
1066static int
1067decorator_names(int ecflags, const char **decorators_ret)
1068{
1069 int num_decorators;
1070
1071 switch (ecflags & ECONV_NEWLINE_DECORATOR_MASK) {
1076 case 0:
1077 break;
1078 default:
1079 return -1;
1080 }
1081
1082 if ((ecflags & ECONV_XML_TEXT_DECORATOR) &&
1084 return -1;
1085
1086 num_decorators = 0;
1087
1088 if (ecflags & ECONV_XML_TEXT_DECORATOR)
1089 decorators_ret[num_decorators++] = "xml_text_escape";
1091 decorators_ret[num_decorators++] = "xml_attr_content_escape";
1092 if (ecflags & ECONV_XML_ATTR_QUOTE_DECORATOR)
1093 decorators_ret[num_decorators++] = "xml_attr_quote";
1094
1095 if (ecflags & ECONV_CRLF_NEWLINE_DECORATOR)
1096 decorators_ret[num_decorators++] = "crlf_newline";
1097 if (ecflags & ECONV_CR_NEWLINE_DECORATOR)
1098 decorators_ret[num_decorators++] = "cr_newline";
1099 if (ecflags & ECONV_LF_NEWLINE_DECORATOR)
1100 decorators_ret[num_decorators++] = "lf_newline";
1102 decorators_ret[num_decorators++] = "universal_newline";
1103
1104 return num_decorators;
1105}
1106
1107rb_econv_t *
1108rb_econv_open(const char *sname, const char *dname, int ecflags)
1109{
1110 rb_econv_t *ec;
1111 int num_decorators;
1112 const char *decorators[MAX_ECFLAGS_DECORATORS];
1113 int i;
1114
1115 num_decorators = decorator_names(ecflags, decorators);
1116 if (num_decorators == -1)
1117 return NULL;
1118
1119 ec = rb_econv_open0(sname, dname, ecflags & ECONV_ERROR_HANDLER_MASK);
1120 if (ec) {
1121 for (i = 0; i < num_decorators; i++) {
1122 if (rb_econv_decorate_at_last(ec, decorators[i]) == -1) {
1123 rb_econv_close(ec);
1124 ec = NULL;
1125 break;
1126 }
1127 }
1128 }
1129
1130 if (ec) {
1131 ec->flags |= ecflags & ~ECONV_ERROR_HANDLER_MASK;
1132 }
1133 return ec; // can be NULL
1134}
1135
1136static int
1137trans_sweep(rb_econv_t *ec,
1138 const unsigned char **input_ptr, const unsigned char *input_stop,
1139 unsigned char **output_ptr, unsigned char *output_stop,
1140 int flags,
1141 int start)
1142{
1143 int try;
1144 int i, f;
1145
1146 const unsigned char **ipp, *is, *iold;
1147 unsigned char **opp, *os, *oold;
1149
1150 try = 1;
1151 while (try) {
1152 try = 0;
1153 for (i = start; i < ec->num_trans; i++) {
1154 rb_econv_elem_t *te = &ec->elems[i];
1155
1156 if (i == 0) {
1157 ipp = input_ptr;
1158 is = input_stop;
1159 }
1160 else {
1161 rb_econv_elem_t *prev_te = &ec->elems[i-1];
1162 ipp = (const unsigned char **)&prev_te->out_data_start;
1163 is = prev_te->out_data_end;
1164 }
1165
1166 if (i == ec->num_trans-1) {
1167 opp = output_ptr;
1168 os = output_stop;
1169 }
1170 else {
1171 if (te->out_buf_start != te->out_data_start) {
1172 ssize_t len = te->out_data_end - te->out_data_start;
1173 ssize_t off = te->out_data_start - te->out_buf_start;
1174 MEMMOVE(te->out_buf_start, te->out_data_start, unsigned char, len);
1175 te->out_data_start = te->out_buf_start;
1176 te->out_data_end -= off;
1177 }
1178 opp = &te->out_data_end;
1179 os = te->out_buf_end;
1180 }
1181
1182 f = flags;
1183 if (ec->num_finished != i)
1185 if (i == 0 && (flags & ECONV_AFTER_OUTPUT)) {
1186 start = 1;
1187 flags &= ~ECONV_AFTER_OUTPUT;
1188 }
1189 if (i != 0)
1191 iold = *ipp;
1192 oold = *opp;
1193 te->last_result = res = rb_transcoding_convert(te->tc, ipp, is, opp, os, f);
1194 if (iold != *ipp || oold != *opp)
1195 try = 1;
1196
1197 switch (res) {
1201 case econv_after_output:
1202 return i;
1203
1206 break;
1207
1208 case econv_finished:
1209 ec->num_finished = i+1;
1210 break;
1211 }
1212 }
1213 }
1214 return -1;
1215}
1216
1217static rb_econv_result_t
1218rb_trans_conv(rb_econv_t *ec,
1219 const unsigned char **input_ptr, const unsigned char *input_stop,
1220 unsigned char **output_ptr, unsigned char *output_stop,
1221 int flags,
1222 int *result_position_ptr)
1223{
1224 int i;
1225 int needreport_index;
1226 int sweep_start;
1227
1228 unsigned char empty_buf;
1229 unsigned char *empty_ptr = &empty_buf;
1230
1231 if (!input_ptr) {
1232 input_ptr = (const unsigned char **)&empty_ptr;
1233 input_stop = empty_ptr;
1234 }
1235
1236 if (!output_ptr) {
1237 output_ptr = &empty_ptr;
1238 output_stop = empty_ptr;
1239 }
1240
1241 if (ec->elems[0].last_result == econv_after_output)
1242 ec->elems[0].last_result = econv_source_buffer_empty;
1243
1244 for (i = ec->num_trans-1; 0 <= i; i--) {
1245 switch (ec->elems[i].last_result) {
1249 case econv_after_output:
1250 case econv_finished:
1251 sweep_start = i+1;
1252 goto found_needreport;
1253
1256 break;
1257
1258 default:
1259 rb_bug("unexpected transcode last result");
1260 }
1261 }
1262
1263 /* /^[sd]+$/ is confirmed. but actually /^s*d*$/. */
1264
1265 if (ec->elems[ec->num_trans-1].last_result == econv_destination_buffer_full &&
1266 (flags & ECONV_AFTER_OUTPUT)) {
1268
1269 res = rb_trans_conv(ec, NULL, NULL, output_ptr, output_stop,
1271 result_position_ptr);
1272
1273 if (res == econv_source_buffer_empty)
1274 return econv_after_output;
1275 return res;
1276 }
1277
1278 sweep_start = 0;
1279
1280 found_needreport:
1281
1282 do {
1283 needreport_index = trans_sweep(ec, input_ptr, input_stop, output_ptr, output_stop, flags, sweep_start);
1284 sweep_start = needreport_index + 1;
1285 } while (needreport_index != -1 && needreport_index != ec->num_trans-1);
1286
1287 for (i = ec->num_trans-1; 0 <= i; i--) {
1288 if (ec->elems[i].last_result != econv_source_buffer_empty) {
1289 rb_econv_result_t res = ec->elems[i].last_result;
1290 if (res == econv_invalid_byte_sequence ||
1291 res == econv_incomplete_input ||
1293 res == econv_after_output) {
1294 ec->elems[i].last_result = econv_source_buffer_empty;
1295 }
1296 if (result_position_ptr)
1297 *result_position_ptr = i;
1298 return res;
1299 }
1300 }
1301 if (result_position_ptr)
1302 *result_position_ptr = -1;
1304}
1305
1306static rb_econv_result_t
1307rb_econv_convert0(rb_econv_t *ec,
1308 const unsigned char **input_ptr, const unsigned char *input_stop,
1309 unsigned char **output_ptr, unsigned char *output_stop,
1310 int flags)
1311{
1313 int result_position;
1314 int has_output = 0;
1315
1316 memset(&ec->last_error, 0, sizeof(ec->last_error));
1317
1318 if (ec->num_trans == 0) {
1319 size_t len;
1320 if (ec->in_buf_start && ec->in_data_start != ec->in_data_end) {
1321 if (output_stop - *output_ptr < ec->in_data_end - ec->in_data_start) {
1322 len = output_stop - *output_ptr;
1323 memcpy(*output_ptr, ec->in_data_start, len);
1324 *output_ptr = output_stop;
1325 ec->in_data_start += len;
1327 goto gotresult;
1328 }
1329 len = ec->in_data_end - ec->in_data_start;
1330 memcpy(*output_ptr, ec->in_data_start, len);
1331 *output_ptr += len;
1332 ec->in_data_start = ec->in_data_end = ec->in_buf_start;
1333 if (flags & ECONV_AFTER_OUTPUT) {
1334 res = econv_after_output;
1335 goto gotresult;
1336 }
1337 }
1338 if (output_stop - *output_ptr < input_stop - *input_ptr) {
1339 len = output_stop - *output_ptr;
1340 }
1341 else {
1342 len = input_stop - *input_ptr;
1343 }
1344 if (0 < len && (flags & ECONV_AFTER_OUTPUT)) {
1345 *(*output_ptr)++ = *(*input_ptr)++;
1346 res = econv_after_output;
1347 goto gotresult;
1348 }
1349 memcpy(*output_ptr, *input_ptr, len);
1350 *output_ptr += len;
1351 *input_ptr += len;
1352 if (*input_ptr != input_stop)
1354 else if (flags & ECONV_PARTIAL_INPUT)
1356 else
1357 res = econv_finished;
1358 goto gotresult;
1359 }
1360
1361 if (ec->elems[ec->num_trans-1].out_data_start) {
1362 unsigned char *data_start = ec->elems[ec->num_trans-1].out_data_start;
1363 unsigned char *data_end = ec->elems[ec->num_trans-1].out_data_end;
1364 if (data_start != data_end) {
1365 size_t len;
1366 if (output_stop - *output_ptr < data_end - data_start) {
1367 len = output_stop - *output_ptr;
1368 memcpy(*output_ptr, data_start, len);
1369 *output_ptr = output_stop;
1370 ec->elems[ec->num_trans-1].out_data_start += len;
1372 goto gotresult;
1373 }
1374 len = data_end - data_start;
1375 memcpy(*output_ptr, data_start, len);
1376 *output_ptr += len;
1377 ec->elems[ec->num_trans-1].out_data_start =
1378 ec->elems[ec->num_trans-1].out_data_end =
1379 ec->elems[ec->num_trans-1].out_buf_start;
1380 has_output = 1;
1381 }
1382 }
1383
1384 if (ec->in_buf_start &&
1385 ec->in_data_start != ec->in_data_end) {
1386 res = rb_trans_conv(ec, (const unsigned char **)&ec->in_data_start, ec->in_data_end, output_ptr, output_stop,
1387 (flags&~ECONV_AFTER_OUTPUT)|ECONV_PARTIAL_INPUT, &result_position);
1388 if (res != econv_source_buffer_empty)
1389 goto gotresult;
1390 }
1391
1392 if (has_output &&
1393 (flags & ECONV_AFTER_OUTPUT) &&
1394 *input_ptr != input_stop) {
1395 input_stop = *input_ptr;
1396 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1397 if (res == econv_source_buffer_empty)
1398 res = econv_after_output;
1399 }
1400 else if ((flags & ECONV_AFTER_OUTPUT) ||
1401 ec->num_trans == 1) {
1402 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1403 }
1404 else {
1405 flags |= ECONV_AFTER_OUTPUT;
1406 do {
1407 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1408 } while (res == econv_after_output);
1409 }
1410
1411 gotresult:
1412 ec->last_error.result = res;
1413 if (res == econv_invalid_byte_sequence ||
1414 res == econv_incomplete_input ||
1416 rb_transcoding *error_tc = ec->elems[result_position].tc;
1417 ec->last_error.error_tc = error_tc;
1418 ec->last_error.source_encoding = error_tc->transcoder->src_encoding;
1419 ec->last_error.destination_encoding = error_tc->transcoder->dst_encoding;
1420 ec->last_error.error_bytes_start = TRANSCODING_READBUF(error_tc);
1421 ec->last_error.error_bytes_len = error_tc->recognized_len;
1422 ec->last_error.readagain_len = error_tc->readagain_len;
1423 }
1424
1425 return res;
1426}
1427
1428static int output_replacement_character(rb_econv_t *ec);
1429
1430static int
1431output_hex_charref(rb_econv_t *ec)
1432{
1433 int ret;
1434 unsigned char utfbuf[1024];
1435 const unsigned char *utf;
1436 size_t utf_len;
1437 int utf_allocated = 0;
1438 char charef_buf[16];
1439 const unsigned char *p;
1440
1441 if (encoding_equal(ec->last_error.source_encoding, "UTF-32BE")) {
1442 utf = ec->last_error.error_bytes_start;
1443 utf_len = ec->last_error.error_bytes_len;
1444 }
1445 else {
1446 utf = allocate_converted_string(ec->last_error.source_encoding, "UTF-32BE",
1447 ec->last_error.error_bytes_start, ec->last_error.error_bytes_len,
1448 utfbuf, sizeof(utfbuf),
1449 &utf_len);
1450 if (!utf)
1451 return -1;
1452 if (utf != utfbuf && utf != ec->last_error.error_bytes_start)
1453 utf_allocated = 1;
1454 }
1455
1456 if (utf_len % 4 != 0)
1457 goto fail;
1458
1459 p = utf;
1460 while (4 <= utf_len) {
1461 unsigned int u = 0;
1462 u += p[0] << 24;
1463 u += p[1] << 16;
1464 u += p[2] << 8;
1465 u += p[3];
1466 snprintf(charef_buf, sizeof(charef_buf), "&#x%X;", u);
1467
1468 ret = rb_econv_insert_output(ec, (unsigned char *)charef_buf, strlen(charef_buf), "US-ASCII");
1469 if (ret == -1)
1470 goto fail;
1471
1472 p += 4;
1473 utf_len -= 4;
1474 }
1475
1476 if (utf_allocated)
1477 xfree((void *)utf);
1478 return 0;
1479
1480 fail:
1481 if (utf_allocated)
1482 xfree((void *)utf);
1483 return -1;
1484}
1485
1488 const unsigned char **input_ptr, const unsigned char *input_stop,
1489 unsigned char **output_ptr, unsigned char *output_stop,
1490 int flags)
1491{
1493
1494 unsigned char empty_buf;
1495 unsigned char *empty_ptr = &empty_buf;
1496
1497 ec->started = 1;
1498
1499 if (!input_ptr) {
1500 input_ptr = (const unsigned char **)&empty_ptr;
1501 input_stop = empty_ptr;
1502 }
1503
1504 if (!output_ptr) {
1505 output_ptr = &empty_ptr;
1506 output_stop = empty_ptr;
1507 }
1508
1509 resume:
1510 ret = rb_econv_convert0(ec, input_ptr, input_stop, output_ptr, output_stop, flags);
1511
1512 if (ret == econv_invalid_byte_sequence ||
1513 ret == econv_incomplete_input) {
1514 /* deal with invalid byte sequence */
1515 /* todo: add more alternative behaviors */
1516 switch (ec->flags & ECONV_INVALID_MASK) {
1518 if (output_replacement_character(ec) == 0)
1519 goto resume;
1520 }
1521 }
1522
1523 if (ret == econv_undefined_conversion) {
1524 /* valid character in source encoding
1525 * but no related character(s) in destination encoding */
1526 /* todo: add more alternative behaviors */
1527 switch (ec->flags & ECONV_UNDEF_MASK) {
1529 if (output_replacement_character(ec) == 0)
1530 goto resume;
1531 break;
1532
1534 if (output_hex_charref(ec) == 0)
1535 goto resume;
1536 break;
1537 }
1538 }
1539
1540 return ret;
1541}
1542
1543const char *
1545{
1546 rb_transcoding *tc = ec->last_tc;
1547 const rb_transcoder *tr;
1548
1549 if (tc == NULL)
1550 return "";
1551
1552 tr = tc->transcoder;
1553
1554 if (tr->asciicompat_type == asciicompat_encoder)
1555 return tr->src_encoding;
1556 return tr->dst_encoding;
1557}
1558
1559static unsigned char *
1560allocate_converted_string(const char *sname, const char *dname,
1561 const unsigned char *str, size_t len,
1562 unsigned char *caller_dst_buf, size_t caller_dst_bufsize,
1563 size_t *dst_len_ptr)
1564{
1565 unsigned char *dst_str;
1566 size_t dst_len;
1567 size_t dst_bufsize;
1568
1569 rb_econv_t *ec;
1571
1572 const unsigned char *sp;
1573 unsigned char *dp;
1574
1575 if (caller_dst_buf)
1576 dst_bufsize = caller_dst_bufsize;
1577 else if (len == 0)
1578 dst_bufsize = 1;
1579 else
1580 dst_bufsize = len;
1581
1582 ec = rb_econv_open(sname, dname, 0);
1583 if (ec == NULL)
1584 return NULL;
1585 if (caller_dst_buf)
1586 dst_str = caller_dst_buf;
1587 else
1588 dst_str = xmalloc(dst_bufsize);
1589 dst_len = 0;
1590 sp = str;
1591 dp = dst_str+dst_len;
1592 res = rb_econv_convert(ec, &sp, str+len, &dp, dst_str+dst_bufsize, 0);
1593 dst_len = dp - dst_str;
1594 while (res == econv_destination_buffer_full) {
1595 if (SIZE_MAX/2 < dst_bufsize) {
1596 goto fail;
1597 }
1598 dst_bufsize *= 2;
1599 if (dst_str == caller_dst_buf) {
1600 unsigned char *tmp;
1601 tmp = xmalloc(dst_bufsize);
1602 memcpy(tmp, dst_str, dst_bufsize/2);
1603 dst_str = tmp;
1604 }
1605 else {
1606 dst_str = xrealloc(dst_str, dst_bufsize);
1607 }
1608 dp = dst_str+dst_len;
1609 res = rb_econv_convert(ec, &sp, str+len, &dp, dst_str+dst_bufsize, 0);
1610 dst_len = dp - dst_str;
1611 }
1612 if (res != econv_finished) {
1613 goto fail;
1614 }
1615 rb_econv_close(ec);
1616 *dst_len_ptr = dst_len;
1617 return dst_str;
1618
1619 fail:
1620 if (dst_str != caller_dst_buf)
1621 xfree(dst_str);
1622 rb_econv_close(ec);
1623 return NULL;
1624}
1625
1626/* result: 0:success -1:failure */
1627int
1629 const unsigned char *str, size_t len, const char *str_encoding)
1630{
1631 const char *insert_encoding = rb_econv_encoding_to_insert_output(ec);
1632 unsigned char insert_buf[4096];
1633 const unsigned char *insert_str = NULL;
1634 size_t insert_len;
1635
1636 int last_trans_index;
1637 rb_transcoding *tc;
1638
1639 unsigned char **buf_start_p;
1640 unsigned char **data_start_p;
1641 unsigned char **data_end_p;
1642 unsigned char **buf_end_p;
1643
1644 size_t need;
1645
1646 ec->started = 1;
1647
1648 if (len == 0)
1649 return 0;
1650
1651 if (encoding_equal(insert_encoding, str_encoding)) {
1652 insert_str = str;
1653 insert_len = len;
1654 }
1655 else {
1656 insert_str = allocate_converted_string(str_encoding, insert_encoding,
1657 str, len, insert_buf, sizeof(insert_buf), &insert_len);
1658 if (insert_str == NULL)
1659 return -1;
1660 }
1661
1662 need = insert_len;
1663
1664 last_trans_index = ec->num_trans-1;
1665 if (ec->num_trans == 0) {
1666 tc = NULL;
1667 buf_start_p = &ec->in_buf_start;
1668 data_start_p = &ec->in_data_start;
1669 data_end_p = &ec->in_data_end;
1670 buf_end_p = &ec->in_buf_end;
1671 }
1672 else if (ec->elems[last_trans_index].tc->transcoder->asciicompat_type == asciicompat_encoder) {
1673 tc = ec->elems[last_trans_index].tc;
1674 need += tc->readagain_len;
1675 if (need < insert_len)
1676 goto fail;
1677 if (last_trans_index == 0) {
1678 buf_start_p = &ec->in_buf_start;
1679 data_start_p = &ec->in_data_start;
1680 data_end_p = &ec->in_data_end;
1681 buf_end_p = &ec->in_buf_end;
1682 }
1683 else {
1684 rb_econv_elem_t *ee = &ec->elems[last_trans_index-1];
1685 buf_start_p = &ee->out_buf_start;
1686 data_start_p = &ee->out_data_start;
1687 data_end_p = &ee->out_data_end;
1688 buf_end_p = &ee->out_buf_end;
1689 }
1690 }
1691 else {
1692 rb_econv_elem_t *ee = &ec->elems[last_trans_index];
1693 buf_start_p = &ee->out_buf_start;
1694 data_start_p = &ee->out_data_start;
1695 data_end_p = &ee->out_data_end;
1696 buf_end_p = &ee->out_buf_end;
1697 tc = ec->elems[last_trans_index].tc;
1698 }
1699
1700 if (*buf_start_p == NULL) {
1701 unsigned char *buf = xmalloc(need);
1702 *buf_start_p = buf;
1703 *data_start_p = buf;
1704 *data_end_p = buf;
1705 *buf_end_p = buf+need;
1706 }
1707 else if ((size_t)(*buf_end_p - *data_end_p) < need) {
1708 MEMMOVE(*buf_start_p, *data_start_p, unsigned char, *data_end_p - *data_start_p);
1709 *data_end_p = *buf_start_p + (*data_end_p - *data_start_p);
1710 *data_start_p = *buf_start_p;
1711 if ((size_t)(*buf_end_p - *data_end_p) < need) {
1712 unsigned char *buf;
1713 size_t s = (*data_end_p - *buf_start_p) + need;
1714 if (s < need)
1715 goto fail;
1716 buf = xrealloc(*buf_start_p, s);
1717 *data_start_p = buf;
1718 *data_end_p = buf + (*data_end_p - *buf_start_p);
1719 *buf_start_p = buf;
1720 *buf_end_p = buf + s;
1721 }
1722 }
1723
1724 memcpy(*data_end_p, insert_str, insert_len);
1725 *data_end_p += insert_len;
1726 if (tc && tc->transcoder->asciicompat_type == asciicompat_encoder) {
1727 memcpy(*data_end_p, TRANSCODING_READBUF(tc)+tc->recognized_len, tc->readagain_len);
1728 *data_end_p += tc->readagain_len;
1729 tc->readagain_len = 0;
1730 }
1731
1732 if (insert_str != str && insert_str != insert_buf)
1733 xfree((void*)insert_str);
1734 return 0;
1735
1736 fail:
1737 if (insert_str != str && insert_str != insert_buf)
1738 xfree((void*)insert_str);
1739 return -1;
1740}
1741
1742void
1744{
1745 int i;
1746
1747 if (ec->replacement_allocated) {
1748 xfree((void *)ec->replacement_str);
1749 }
1750 for (i = 0; i < ec->num_trans; i++) {
1751 rb_transcoding_close(ec->elems[i].tc);
1752 xfree(ec->elems[i].out_buf_start);
1753 }
1754 xfree(ec->in_buf_start);
1755 xfree(ec->elems);
1756 xfree(ec);
1757}
1758
1759size_t
1760rb_econv_memsize(rb_econv_t *ec)
1761{
1762 size_t size = sizeof(rb_econv_t);
1763 int i;
1764
1765 if (ec->replacement_allocated) {
1766 size += ec->replacement_len;
1767 }
1768 for (i = 0; i < ec->num_trans; i++) {
1769 size += rb_transcoding_memsize(ec->elems[i].tc);
1770
1771 if (ec->elems[i].out_buf_start) {
1772 size += ec->elems[i].out_buf_end - ec->elems[i].out_buf_start;
1773 }
1774 }
1775 size += ec->in_buf_end - ec->in_buf_start;
1776 size += sizeof(rb_econv_elem_t) * ec->num_allocated;
1777
1778 return size;
1779}
1780
1781int
1783{
1784 if (ec->num_trans == 0)
1785 return 0;
1786#if SIZEOF_SIZE_T > SIZEOF_INT
1787 if (ec->elems[0].tc->readagain_len > INT_MAX) return INT_MAX;
1788#endif
1789 return (int)ec->elems[0].tc->readagain_len;
1790}
1791
1792void
1793rb_econv_putback(rb_econv_t *ec, unsigned char *p, int n)
1794{
1795 rb_transcoding *tc;
1796 if (ec->num_trans == 0 || n == 0)
1797 return;
1798 tc = ec->elems[0].tc;
1799 memcpy(p, TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len - n, n);
1800 tc->readagain_len -= n;
1801}
1802
1804 const char *ascii_compat_name;
1805 const char *ascii_incompat_name;
1806};
1807
1808static int
1809asciicompat_encoding_i(st_data_t key, st_data_t val, st_data_t arg)
1810{
1811 struct asciicompat_encoding_t *data = (struct asciicompat_encoding_t *)arg;
1812 transcoder_entry_t *entry = (transcoder_entry_t *)val;
1813 const rb_transcoder *tr;
1814
1815 if (DECORATOR_P(entry->sname, entry->dname))
1816 return ST_CONTINUE;
1817 tr = load_transcoder_entry(entry);
1818 if (tr && tr->asciicompat_type == asciicompat_decoder) {
1819 data->ascii_compat_name = tr->dst_encoding;
1820 return ST_STOP;
1821 }
1822 return ST_CONTINUE;
1823}
1824
1825const char *
1826rb_econv_asciicompat_encoding(const char *ascii_incompat_name)
1827{
1828 st_data_t v;
1829 st_table *table2;
1830 struct asciicompat_encoding_t data = {0};
1831
1832 unsigned int lev;
1833 RB_VM_LOCK_ENTER_LEV(&lev);
1834 {
1835 if (st_lookup(transcoder_table, (st_data_t)ascii_incompat_name, &v)) {
1836 table2 = (st_table *)v;
1837 /*
1838 * Assumption:
1839 * There is at most one transcoder for
1840 * converting from ASCII incompatible encoding.
1841 *
1842 * For ISO-2022-JP, there is ISO-2022-JP -> stateless-ISO-2022-JP and no others.
1843 */
1844 if (table2->num_entries == 1) {
1845 data.ascii_incompat_name = ascii_incompat_name;
1846 data.ascii_compat_name = NULL;
1847 if (rb_multi_ractor_p()) {
1848 /*
1849 * We need to unlock in case `load_transcoder_entry` actually loads the encoding
1850 * and table2 could be inserted into when we unlock.
1851 */
1852 st_table *dup_table2 = st_copy(table2);
1853 RB_VM_LOCK_LEAVE_LEV(&lev);
1854 st_foreach(dup_table2, asciicompat_encoding_i, (st_data_t)&data);
1855 st_free_table(dup_table2);
1856 RB_VM_LOCK_ENTER_LEV(&lev);
1857 }
1858 else {
1859 st_foreach(table2, asciicompat_encoding_i, (st_data_t)&data);
1860 }
1861 }
1862
1863 }
1864 }
1865 RB_VM_LOCK_LEAVE_LEV(&lev);
1866
1867 return data.ascii_compat_name; // can be NULL
1868}
1869
1870/*
1871 * Append `len` bytes pointed by `ss` to `dst` with converting with `ec`.
1872 *
1873 * If the result of the conversion is not compatible with the encoding of
1874 * `dst`, `dst` may not be valid encoding.
1875 */
1876VALUE
1877rb_econv_append(rb_econv_t *ec, const char *ss, long len, VALUE dst, int flags)
1878{
1879 unsigned const char *sp, *se;
1880 unsigned char *ds, *dp, *de;
1882 int max_output;
1883 enum ruby_coderange_type coderange;
1884 rb_encoding *dst_enc = ec->destination_encoding;
1885
1886 if (NIL_P(dst)) {
1887 dst = rb_str_buf_new(len);
1888 if (dst_enc) {
1889 rb_enc_associate(dst, dst_enc);
1890 }
1891 coderange = ENC_CODERANGE_7BIT; // scan from the start
1892 }
1893 else {
1894 dst_enc = rb_enc_get(dst);
1895 coderange = rb_enc_str_coderange(dst);
1896 }
1897
1898 if (ec->last_tc)
1899 max_output = ec->last_tc->transcoder->max_output;
1900 else
1901 max_output = 1;
1902
1903 do {
1904 int cr;
1905 long dlen = RSTRING_LEN(dst);
1906 if (rb_str_capacity(dst) - dlen < (size_t)len + max_output) {
1907 unsigned long new_capa = (unsigned long)dlen + len + max_output;
1908 if (LONG_MAX < new_capa)
1909 rb_raise(rb_eArgError, "too long string");
1910 rb_str_modify_expand(dst, new_capa - dlen);
1911 }
1912 sp = (const unsigned char *)ss;
1913 se = sp + len;
1914 ds = (unsigned char *)RSTRING_PTR(dst);
1915 de = ds + rb_str_capacity(dst);
1916 dp = ds += dlen;
1917 res = rb_econv_convert(ec, &sp, se, &dp, de, flags);
1918 switch (coderange) {
1919 case ENC_CODERANGE_7BIT:
1921 cr = (int)coderange;
1922 rb_str_coderange_scan_restartable((char *)ds, (char *)dp, dst_enc, &cr);
1923 coderange = cr;
1924 ENC_CODERANGE_SET(dst, coderange);
1925 break;
1928 break;
1929 }
1930 len -= (const char *)sp - ss;
1931 ss = (const char *)sp;
1932 rb_str_set_len(dst, dlen + (dp - ds));
1934 } while (res == econv_destination_buffer_full);
1935
1936 return dst;
1937}
1938
1939VALUE
1940rb_econv_substr_append(rb_econv_t *ec, VALUE src, long off, long len, VALUE dst, int flags)
1941{
1942 src = rb_str_new_frozen(src);
1943 dst = rb_econv_append(ec, RSTRING_PTR(src) + off, len, dst, flags);
1944 RB_GC_GUARD(src);
1945 return dst;
1946}
1947
1948VALUE
1950{
1951 return rb_econv_substr_append(ec, src, 0, RSTRING_LEN(src), dst, flags);
1952}
1953
1954VALUE
1955rb_econv_substr_convert(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, int flags)
1956{
1957 return rb_econv_substr_append(ec, src, byteoff, bytesize, Qnil, flags);
1958}
1959
1960VALUE
1962{
1963 return rb_econv_substr_append(ec, src, 0, RSTRING_LEN(src), Qnil, flags);
1964}
1965
1966static int
1967rb_econv_add_converter(rb_econv_t *ec, const char *sname, const char *dname, int n)
1968{
1969 transcoder_entry_t *entry;
1970 const rb_transcoder *tr = NULL;
1971
1972 if (ec->started != 0)
1973 return -1;
1974
1975 entry = get_transcoder_entry(sname, dname);
1976 if (entry) {
1977 tr = load_transcoder_entry(entry);
1978 }
1979
1980 return tr ? rb_econv_add_transcoder_at(ec, tr, n) : -1;
1981}
1982
1983static int
1984rb_econv_decorate_at(rb_econv_t *ec, const char *decorator_name, int n)
1985{
1986 return rb_econv_add_converter(ec, "", decorator_name, n);
1987}
1988
1989int
1990rb_econv_decorate_at_first(rb_econv_t *ec, const char *decorator_name)
1991{
1992 const rb_transcoder *tr;
1993
1994 if (ec->num_trans == 0)
1995 return rb_econv_decorate_at(ec, decorator_name, 0);
1996
1997 tr = ec->elems[0].tc->transcoder;
1998
1999 if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding) &&
2000 tr->asciicompat_type == asciicompat_decoder)
2001 return rb_econv_decorate_at(ec, decorator_name, 1);
2002
2003 return rb_econv_decorate_at(ec, decorator_name, 0);
2004}
2005
2006int
2007rb_econv_decorate_at_last(rb_econv_t *ec, const char *decorator_name)
2008{
2009 const rb_transcoder *tr;
2010
2011 if (ec->num_trans == 0)
2012 return rb_econv_decorate_at(ec, decorator_name, 0);
2013
2014 tr = ec->elems[ec->num_trans-1].tc->transcoder;
2015
2016 if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding) &&
2017 tr->asciicompat_type == asciicompat_encoder)
2018 return rb_econv_decorate_at(ec, decorator_name, ec->num_trans-1);
2019
2020 return rb_econv_decorate_at(ec, decorator_name, ec->num_trans);
2021}
2022
2023void
2025{
2026 const char *dname = 0;
2027
2028 switch (ec->flags & ECONV_NEWLINE_DECORATOR_MASK) {
2030 dname = "universal_newline";
2031 break;
2033 dname = "crlf_newline";
2034 break;
2036 dname = "cr_newline";
2037 break;
2039 dname = "lf_newline";
2040 break;
2041 }
2042
2043 if (dname) {
2044 const rb_transcoder *transcoder = get_transcoder_entry("", dname)->transcoder;
2045 int num_trans = ec->num_trans;
2046 int i, j = 0;
2047
2048 for (i=0; i < num_trans; i++) {
2049 if (transcoder == ec->elems[i].tc->transcoder) {
2050 rb_transcoding_close(ec->elems[i].tc);
2051 xfree(ec->elems[i].out_buf_start);
2052 ec->num_trans--;
2053 }
2054 else
2055 ec->elems[j++] = ec->elems[i];
2056 }
2057 }
2058
2059 ec->flags &= ~ECONV_NEWLINE_DECORATOR_MASK;
2060}
2061
2062static VALUE
2063econv_description(const char *sname, const char *dname, int ecflags, VALUE mesg)
2064{
2065 int has_description = 0;
2066
2067 if (NIL_P(mesg))
2068 mesg = rb_str_new(NULL, 0);
2069
2070 if (*sname != '\0' || *dname != '\0') {
2071 if (*sname == '\0')
2072 rb_str_cat2(mesg, dname);
2073 else if (*dname == '\0')
2074 rb_str_cat2(mesg, sname);
2075 else
2076 rb_str_catf(mesg, "%s to %s", sname, dname);
2077 has_description = 1;
2078 }
2079
2080 if (ecflags & (ECONV_NEWLINE_DECORATOR_MASK|
2084 const char *pre = "";
2085 if (has_description)
2086 rb_str_cat2(mesg, " with ");
2087 if (ecflags & ECONV_UNIVERSAL_NEWLINE_DECORATOR) {
2088 rb_str_cat2(mesg, pre); pre = ",";
2089 rb_str_cat2(mesg, "universal_newline");
2090 }
2091 if (ecflags & ECONV_CRLF_NEWLINE_DECORATOR) {
2092 rb_str_cat2(mesg, pre); pre = ",";
2093 rb_str_cat2(mesg, "crlf_newline");
2094 }
2095 if (ecflags & ECONV_CR_NEWLINE_DECORATOR) {
2096 rb_str_cat2(mesg, pre); pre = ",";
2097 rb_str_cat2(mesg, "cr_newline");
2098 }
2099 if (ecflags & ECONV_LF_NEWLINE_DECORATOR) {
2100 rb_str_cat2(mesg, pre); pre = ",";
2101 rb_str_cat2(mesg, "lf_newline");
2102 }
2103 if (ecflags & ECONV_XML_TEXT_DECORATOR) {
2104 rb_str_cat2(mesg, pre); pre = ",";
2105 rb_str_cat2(mesg, "xml_text");
2106 }
2107 if (ecflags & ECONV_XML_ATTR_CONTENT_DECORATOR) {
2108 rb_str_cat2(mesg, pre); pre = ",";
2109 rb_str_cat2(mesg, "xml_attr_content");
2110 }
2111 if (ecflags & ECONV_XML_ATTR_QUOTE_DECORATOR) {
2112 rb_str_cat2(mesg, pre); pre = ",";
2113 rb_str_cat2(mesg, "xml_attr_quote");
2114 }
2115 has_description = 1;
2116 }
2117 if (!has_description) {
2118 rb_str_cat2(mesg, "no-conversion");
2119 }
2120
2121 return mesg;
2122}
2123
2124VALUE
2125rb_econv_open_exc(const char *sname, const char *dname, int ecflags)
2126{
2127 VALUE mesg, exc;
2128 mesg = rb_str_new_cstr("code converter not found (");
2129 econv_description(sname, dname, ecflags, mesg);
2130 rb_str_cat2(mesg, ")");
2131 exc = rb_exc_new3(rb_eConverterNotFoundError, mesg);
2132 return exc;
2133}
2134
2135static VALUE
2136make_econv_exception(rb_econv_t *ec)
2137{
2138 VALUE mesg, exc;
2139 if (ec->last_error.result == econv_invalid_byte_sequence ||
2140 ec->last_error.result == econv_incomplete_input) {
2141 const char *err = (const char *)ec->last_error.error_bytes_start;
2142 size_t error_len = ec->last_error.error_bytes_len;
2143 VALUE bytes = rb_str_new(err, error_len);
2144 VALUE dumped = rb_str_dump(bytes);
2145 size_t readagain_len = ec->last_error.readagain_len;
2146 VALUE bytes2 = Qnil;
2147 VALUE dumped2;
2148 if (ec->last_error.result == econv_incomplete_input) {
2149 mesg = rb_sprintf("incomplete %s on %s",
2150 StringValueCStr(dumped),
2151 ec->last_error.source_encoding);
2152 }
2153 else if (readagain_len) {
2154 bytes2 = rb_str_new(err+error_len, readagain_len);
2155 dumped2 = rb_str_dump(bytes2);
2156 mesg = rb_sprintf("%s followed by %s on %s",
2157 StringValueCStr(dumped),
2158 StringValueCStr(dumped2),
2159 ec->last_error.source_encoding);
2160 }
2161 else {
2162 mesg = rb_sprintf("%s on %s",
2163 StringValueCStr(dumped),
2164 ec->last_error.source_encoding);
2165 }
2166
2167 exc = rb_exc_new3(rb_eInvalidByteSequenceError, mesg);
2168 rb_ivar_set(exc, id_error_bytes, bytes);
2169 rb_ivar_set(exc, id_readagain_bytes, bytes2);
2170 rb_ivar_set(exc, id_incomplete_input, RBOOL(ec->last_error.result == econv_incomplete_input));
2171 goto set_encs;
2172 }
2173 if (ec->last_error.result == econv_undefined_conversion) {
2174 VALUE bytes = rb_str_new((const char *)ec->last_error.error_bytes_start,
2175 ec->last_error.error_bytes_len);
2176 VALUE dumped = Qnil;
2177 int idx;
2178 if (strcmp(ec->last_error.source_encoding, "UTF-8") == 0) {
2179 rb_encoding *utf8 = rb_utf8_encoding();
2180 const char *start, *end;
2181 int n;
2182 start = (const char *)ec->last_error.error_bytes_start;
2183 end = start + ec->last_error.error_bytes_len;
2184 n = rb_enc_precise_mbclen(start, end, utf8);
2185 if (MBCLEN_CHARFOUND_P(n) &&
2186 (size_t)MBCLEN_CHARFOUND_LEN(n) == ec->last_error.error_bytes_len) {
2187 unsigned int cc = rb_enc_mbc_to_codepoint(start, end, utf8);
2188 dumped = rb_sprintf("U+%04X", cc);
2189 }
2190 }
2191 if (NIL_P(dumped))
2192 dumped = rb_str_dump(bytes);
2193 if (strcmp(ec->last_error.source_encoding,
2194 ec->source_encoding_name) == 0 &&
2195 strcmp(ec->last_error.destination_encoding,
2196 ec->destination_encoding_name) == 0) {
2197 mesg = rb_sprintf("%s from %s to %s",
2198 StringValueCStr(dumped),
2199 ec->last_error.source_encoding,
2200 ec->last_error.destination_encoding);
2201 }
2202 else {
2203 int i;
2204 mesg = rb_sprintf("%s to %s in conversion from %s",
2205 StringValueCStr(dumped),
2206 ec->last_error.destination_encoding,
2207 ec->source_encoding_name);
2208 for (i = 0; i < ec->num_trans; i++) {
2209 const rb_transcoder *tr = ec->elems[i].tc->transcoder;
2210 if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding))
2211 rb_str_catf(mesg, " to %s",
2212 ec->elems[i].tc->transcoder->dst_encoding);
2213 }
2214 }
2215 exc = rb_exc_new3(rb_eUndefinedConversionError, mesg);
2216 idx = rb_enc_find_index(ec->last_error.source_encoding);
2217 if (0 <= idx)
2218 rb_enc_associate_index(bytes, idx);
2219 rb_ivar_set(exc, id_error_char, bytes);
2220 goto set_encs;
2221 }
2222 return Qnil;
2223
2224 set_encs:
2225 rb_ivar_set(exc, id_source_encoding_name, rb_str_new2(ec->last_error.source_encoding));
2226 rb_ivar_set(exc, id_destination_encoding_name, rb_str_new2(ec->last_error.destination_encoding));
2227 int idx = rb_enc_find_index(ec->last_error.source_encoding);
2228 if (0 <= idx)
2229 rb_ivar_set(exc, id_source_encoding, rb_enc_from_encoding(rb_enc_from_index(idx)));
2230 idx = rb_enc_find_index(ec->last_error.destination_encoding);
2231 if (0 <= idx)
2232 rb_ivar_set(exc, id_destination_encoding, rb_enc_from_encoding(rb_enc_from_index(idx)));
2233 return exc;
2234}
2235
2236static void
2237more_output_buffer(
2238 VALUE destination,
2239 unsigned char *(*resize_destination)(VALUE, size_t, size_t),
2240 int max_output,
2241 unsigned char **out_start_ptr,
2242 unsigned char **out_pos,
2243 unsigned char **out_stop_ptr)
2244{
2245 size_t len = (*out_pos - *out_start_ptr);
2246 if (len > (size_t)(LONG_MAX / 2 - max_output))
2247 rb_raise(rb_eArgError, "string size too big");
2248 size_t new_len = (len + max_output) * 2;
2249 *out_start_ptr = resize_destination(destination, len, new_len);
2250 *out_pos = *out_start_ptr + len;
2251 *out_stop_ptr = *out_start_ptr + new_len;
2252}
2253
2254static int
2255make_replacement(rb_econv_t *ec)
2256{
2257 rb_transcoding *tc;
2258 const rb_transcoder *tr;
2259 const unsigned char *replacement;
2260 const char *repl_enc;
2261 const char *ins_enc;
2262 size_t len;
2263
2264 if (ec->replacement_str)
2265 return 0;
2266
2268
2269 tc = ec->last_tc;
2270 if (*ins_enc) {
2271 tr = tc->transcoder;
2272 rb_enc_find(tr->dst_encoding);
2273 replacement = (const unsigned char *)get_replacement_character(ins_enc, &len, &repl_enc);
2274 }
2275 else {
2276 replacement = (unsigned char *)"?";
2277 len = 1;
2278 repl_enc = "";
2279 }
2280
2281 ec->replacement_str = replacement;
2282 ec->replacement_len = len;
2283 ec->replacement_enc = repl_enc;
2284 ec->replacement_allocated = 0;
2285 return 0;
2286}
2287
2288int
2290 const unsigned char *str, size_t len, const char *encname)
2291{
2292 unsigned char *str2;
2293 size_t len2;
2294 const char *encname2;
2295
2297
2298 if (!*encname2 || encoding_equal(encname, encname2)) {
2299 str2 = xmalloc(len);
2300 MEMCPY(str2, str, unsigned char, len); /* xxx: str may be invalid */
2301 len2 = len;
2302 encname2 = encname;
2303 }
2304 else {
2305 str2 = allocate_converted_string(encname, encname2, str, len, NULL, 0, &len2);
2306 if (!str2)
2307 return -1;
2308 }
2309
2310 if (ec->replacement_allocated) {
2311 xfree((void *)ec->replacement_str);
2312 }
2313 ec->replacement_allocated = 1;
2314 ec->replacement_str = str2;
2315 ec->replacement_len = len2;
2316 ec->replacement_enc = encname2;
2317 return 0;
2318}
2319
2320static int
2321output_replacement_character(rb_econv_t *ec)
2322{
2323 int ret;
2324
2325 if (make_replacement(ec) == -1)
2326 return -1;
2327
2328 ret = rb_econv_insert_output(ec, ec->replacement_str, ec->replacement_len, ec->replacement_enc);
2329 if (ret == -1)
2330 return -1;
2331
2332 return 0;
2333}
2334
2335#if 1
2336#define hash_fallback rb_hash_aref
2337
2338static VALUE
2339proc_fallback(VALUE fallback, VALUE c)
2340{
2341 return rb_proc_call(fallback, rb_ary_new4(1, &c));
2342}
2343
2344static VALUE
2345method_fallback(VALUE fallback, VALUE c)
2346{
2347 return rb_method_call(1, &c, fallback);
2348}
2349
2350static VALUE
2351aref_fallback(VALUE fallback, VALUE c)
2352{
2353 return rb_funcallv_public(fallback, idAREF, 1, &c);
2354}
2355
2357 VALUE (*fallback_func)(VALUE, VALUE);
2358 VALUE fallback;
2359 VALUE rep;
2360};
2361
2362static VALUE
2363transcode_loop_fallback_try(VALUE a)
2364{
2366
2367 VALUE ret = args->fallback_func(args->fallback, args->rep);
2368
2369 if (!UNDEF_P(ret) && !NIL_P(ret)) {
2370 StringValue(ret);
2371 }
2372
2373 return ret;
2374}
2375
2376static void
2377transcode_loop(const unsigned char **in_pos, unsigned char **out_pos,
2378 const unsigned char *in_stop, unsigned char *out_stop,
2379 VALUE destination,
2380 unsigned char *(*resize_destination)(VALUE, size_t, size_t),
2381 const char *src_encoding,
2382 const char *dst_encoding,
2383 int ecflags,
2384 VALUE ecopts,
2385 VALUE source)
2386{
2387 rb_econv_t *ec;
2388 rb_transcoding *last_tc;
2390 unsigned char *out_start = *out_pos;
2391 int max_output;
2392 VALUE exc;
2393 VALUE fallback = Qnil;
2394 VALUE (*fallback_func)(VALUE, VALUE) = 0;
2395 const unsigned char *source_start = *in_pos;
2396 long source_len = in_stop - *in_pos;
2397
2398 ec = rb_econv_open_opts(src_encoding, dst_encoding, ecflags, ecopts);
2399 if (!ec)
2400 rb_exc_raise(rb_econv_open_exc(src_encoding, dst_encoding, ecflags));
2401
2402 if (!NIL_P(ecopts) && RB_TYPE_P(ecopts, T_HASH)) {
2403 fallback = rb_hash_aref(ecopts, sym_fallback);
2404 if (RB_TYPE_P(fallback, T_HASH)) {
2405 fallback_func = hash_fallback;
2406 }
2407 else if (rb_obj_is_proc(fallback)) {
2408 fallback_func = proc_fallback;
2409 }
2410 else if (rb_obj_is_method(fallback)) {
2411 fallback_func = method_fallback;
2412 }
2413 else {
2414 fallback_func = aref_fallback;
2415 }
2416 }
2417 last_tc = ec->last_tc;
2418 max_output = last_tc ? last_tc->transcoder->max_output : 1;
2419
2420 resume:
2421 ret = rb_econv_convert(ec, in_pos, in_stop, out_pos, out_stop, 0);
2422
2423 if (!NIL_P(fallback) && ret == econv_undefined_conversion) {
2424 VALUE rep = rb_enc_str_new(
2425 (const char *)ec->last_error.error_bytes_start,
2426 ec->last_error.error_bytes_len,
2427 rb_enc_find(ec->last_error.source_encoding));
2428
2429
2430 struct transcode_loop_fallback_args args = {
2431 .fallback_func = fallback_func,
2432 .fallback = fallback,
2433 .rep = rep,
2434 };
2435
2436 int state;
2437 rep = rb_protect(transcode_loop_fallback_try, (VALUE)&args, &state);
2438 if (state) {
2439 rb_econv_close(ec);
2440 rb_jump_tag(state);
2441 }
2442
2443 /* Ruby code run during the conversion (e.g. the fallback) may have
2444 * modified the source string, invalidating the pointers into its
2445 * buffer. */
2446 if ((const unsigned char *)RSTRING_PTR(source) != source_start ||
2447 RSTRING_LEN(source) != source_len) {
2448 rb_econv_close(ec);
2449 rb_raise(rb_eRuntimeError, "string modified");
2450 }
2451
2452 if (!UNDEF_P(rep) && !NIL_P(rep)) {
2453 ret = rb_econv_insert_output(ec, (const unsigned char *)RSTRING_PTR(rep),
2454 RSTRING_LEN(rep), rb_enc_name(rb_enc_get(rep)));
2455 RB_GC_GUARD(rep); // insert_output may GC while reading rep's bytes
2456 if ((int)ret == -1) {
2457 rb_econv_close(ec);
2458 rb_raise(rb_eArgError, "too big fallback string");
2459 }
2460 goto resume;
2461 }
2462 }
2463
2464 if (ret == econv_invalid_byte_sequence ||
2465 ret == econv_incomplete_input ||
2467 exc = make_econv_exception(ec);
2468 rb_econv_close(ec);
2469 rb_exc_raise(exc);
2470 }
2471
2472 if (ret == econv_destination_buffer_full) {
2473 more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop);
2474 goto resume;
2475 }
2476
2477 rb_econv_close(ec);
2478 return;
2479}
2480#else
2481/* sample transcode_loop implementation in byte-by-byte stream style */
2482static void
2483transcode_loop(const unsigned char **in_pos, unsigned char **out_pos,
2484 const unsigned char *in_stop, unsigned char *out_stop,
2485 VALUE destination,
2486 unsigned char *(*resize_destination)(VALUE, size_t, size_t),
2487 const char *src_encoding,
2488 const char *dst_encoding,
2489 int ecflags,
2490 VALUE ecopts,
2491 VALUE source)
2492{
2493 rb_econv_t *ec;
2494 rb_transcoding *last_tc;
2496 unsigned char *out_start = *out_pos;
2497 const unsigned char *ptr;
2498 int max_output;
2499 VALUE exc;
2500
2501 ec = rb_econv_open_opts(src_encoding, dst_encoding, ecflags, ecopts);
2502 if (!ec)
2503 rb_exc_raise(rb_econv_open_exc(src_encoding, dst_encoding, ecflags));
2504
2505 last_tc = ec->last_tc;
2506 max_output = last_tc ? last_tc->transcoder->max_output : 1;
2507
2509 ptr = *in_pos;
2510 while (ret != econv_finished) {
2511 unsigned char input_byte;
2512 const unsigned char *p = &input_byte;
2513
2514 if (ret == econv_source_buffer_empty) {
2515 if (ptr < in_stop) {
2516 input_byte = *ptr;
2517 ret = rb_econv_convert(ec, &p, p+1, out_pos, out_stop, ECONV_PARTIAL_INPUT);
2518 }
2519 else {
2520 ret = rb_econv_convert(ec, NULL, NULL, out_pos, out_stop, 0);
2521 }
2522 }
2523 else {
2524 ret = rb_econv_convert(ec, NULL, NULL, out_pos, out_stop, ECONV_PARTIAL_INPUT);
2525 }
2526 if (&input_byte != p)
2527 ptr += p - &input_byte;
2528 switch (ret) {
2532 exc = make_econv_exception(ec);
2533 rb_econv_close(ec);
2534 rb_exc_raise(exc);
2535 break;
2536
2538 more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop);
2539 break;
2540
2542 break;
2543
2544 case econv_finished:
2545 break;
2546 }
2547 }
2548 rb_econv_close(ec);
2549 *in_pos = in_stop;
2550 return;
2551}
2552#endif
2553
2554
2555/*
2556 * String-specific code
2557 */
2558
2559static unsigned char *
2560str_transcoding_resize(VALUE destination, size_t len, size_t new_len)
2561{
2562 rb_str_resize(destination, new_len);
2563 return (unsigned char *)RSTRING_PTR(destination);
2564}
2565
2566static int
2567econv_opts(VALUE opt, int ecflags)
2568{
2569 VALUE v;
2570 int newlineflag = 0;
2571
2572 v = rb_hash_aref(opt, sym_invalid);
2573 if (NIL_P(v)) {
2574 }
2575 else if (v==sym_replace) {
2576 ecflags |= ECONV_INVALID_REPLACE;
2577 }
2578 else {
2579 rb_raise(rb_eArgError, "unknown value for invalid character option");
2580 }
2581
2582 v = rb_hash_aref(opt, sym_undef);
2583 if (NIL_P(v)) {
2584 }
2585 else if (v==sym_replace) {
2586 ecflags |= ECONV_UNDEF_REPLACE;
2587 }
2588 else {
2589 rb_raise(rb_eArgError, "unknown value for undefined character option");
2590 }
2591
2592 v = rb_hash_aref(opt, sym_replace);
2593 if (!NIL_P(v) && !(ecflags & ECONV_INVALID_REPLACE)) {
2594 ecflags |= ECONV_UNDEF_REPLACE;
2595 }
2596
2597 v = rb_hash_aref(opt, sym_xml);
2598 if (!NIL_P(v)) {
2599 if (v==sym_text) {
2601 }
2602 else if (v==sym_attr) {
2604 }
2605 else if (SYMBOL_P(v)) {
2606 rb_raise(rb_eArgError, "unexpected value for xml option: %"PRIsVALUE, rb_sym2str(v));
2607 }
2608 else {
2609 rb_raise(rb_eArgError, "unexpected value for xml option");
2610 }
2611 }
2612
2613#ifdef ENABLE_ECONV_NEWLINE_OPTION
2614 v = rb_hash_aref(opt, sym_newline);
2615 if (!NIL_P(v)) {
2616 newlineflag = 2;
2618 if (v == sym_universal) {
2620 }
2621 else if (v == sym_crlf) {
2623 }
2624 else if (v == sym_cr) {
2625 ecflags |= ECONV_CR_NEWLINE_DECORATOR;
2626 }
2627 else if (v == sym_lf) {
2628 ecflags |= ECONV_LF_NEWLINE_DECORATOR;
2629 }
2630 else if (SYMBOL_P(v)) {
2631 rb_raise(rb_eArgError, "unexpected value for newline option: %"PRIsVALUE,
2632 rb_sym2str(v));
2633 }
2634 else {
2635 rb_raise(rb_eArgError, "unexpected value for newline option");
2636 }
2637 }
2638#endif
2639 {
2640 int setflags = 0;
2641
2642 v = rb_hash_aref(opt, sym_universal_newline);
2643 if (RTEST(v))
2645 newlineflag |= !NIL_P(v);
2646
2647 v = rb_hash_aref(opt, sym_crlf_newline);
2648 if (RTEST(v))
2649 setflags |= ECONV_CRLF_NEWLINE_DECORATOR;
2650 newlineflag |= !NIL_P(v);
2651
2652 v = rb_hash_aref(opt, sym_cr_newline);
2653 if (RTEST(v))
2654 setflags |= ECONV_CR_NEWLINE_DECORATOR;
2655 newlineflag |= !NIL_P(v);
2656
2657 v = rb_hash_aref(opt, sym_lf_newline);
2658 if (RTEST(v))
2659 setflags |= ECONV_LF_NEWLINE_DECORATOR;
2660 newlineflag |= !NIL_P(v);
2661
2662 switch (newlineflag) {
2663 case 1:
2665 ecflags |= setflags;
2666 break;
2667
2668 case 3:
2669 rb_warning(":newline option precedes other newline options");
2670 break;
2671 }
2672 }
2673
2674 return ecflags;
2675}
2676
2677int
2678rb_econv_prepare_options(VALUE opthash, VALUE *opts, int ecflags)
2679{
2680 VALUE newhash = Qnil;
2681 VALUE v;
2682
2683 if (NIL_P(opthash)) {
2684 *opts = Qnil;
2685 return ecflags;
2686 }
2687 ecflags = econv_opts(opthash, ecflags);
2688
2689 v = rb_hash_aref(opthash, sym_replace);
2690 if (!NIL_P(v)) {
2691 StringValue(v);
2692 if (is_broken_string(v)) {
2693 VALUE dumped = rb_str_dump(v);
2694 rb_raise(rb_eArgError, "replacement string is broken: %s as %s",
2695 StringValueCStr(dumped),
2696 rb_enc_name(rb_enc_get(v)));
2697 }
2698 v = rb_str_new_frozen(v);
2699 newhash = rb_hash_new();
2700 rb_hash_aset(newhash, sym_replace, v);
2701 }
2702
2703 v = rb_hash_aref(opthash, sym_fallback);
2704 if (!NIL_P(v)) {
2705 VALUE h = rb_check_hash_type(v);
2706 if (NIL_P(h)
2707 ? (rb_obj_is_proc(v) || rb_obj_is_method(v) || rb_respond_to(v, idAREF))
2708 : (v = h, 1)) {
2709 if (NIL_P(newhash))
2710 newhash = rb_hash_new();
2711 rb_hash_aset(newhash, sym_fallback, v);
2712 }
2713 }
2714
2715 if (!NIL_P(newhash))
2716 rb_hash_freeze(newhash);
2717 *opts = newhash;
2718
2719 return ecflags;
2720}
2721
2722int
2724{
2725 return rb_econv_prepare_options(opthash, opts, 0);
2726}
2727
2728rb_econv_t *
2729rb_econv_open_opts(const char *source_encoding, const char *destination_encoding, int ecflags, VALUE opthash)
2730{
2731 rb_econv_t *ec;
2732 VALUE replacement;
2733
2734 if (NIL_P(opthash)) {
2735 replacement = Qnil;
2736 }
2737 else {
2738 if (!RB_TYPE_P(opthash, T_HASH) || !OBJ_FROZEN(opthash))
2739 rb_bug("rb_econv_open_opts called with invalid opthash");
2740 replacement = rb_hash_aref(opthash, sym_replace);
2741 }
2742
2743 ec = rb_econv_open(source_encoding, destination_encoding, ecflags);
2744 if (ec) {
2745 if (!NIL_P(replacement)) {
2746 int ret;
2747 rb_encoding *enc = rb_enc_get(replacement);
2748
2749 ret = rb_econv_set_replacement(ec,
2750 (const unsigned char *)RSTRING_PTR(replacement),
2751 RSTRING_LEN(replacement),
2752 rb_enc_name(enc));
2753 if (ret == -1) {
2754 rb_econv_close(ec);
2755 ec = NULL;
2756 }
2757 }
2758 }
2759 return ec; // can be NULL
2760}
2761
2762static int
2763enc_arg(VALUE *arg, const char **name_p, rb_encoding **enc_p)
2764{
2765 rb_encoding *enc;
2766 const char *n;
2767 int encidx;
2768 VALUE encval;
2769
2770 if (((encidx = rb_to_encoding_index(encval = *arg)) < 0) ||
2771 !(enc = rb_enc_from_index(encidx))) {
2772 enc = NULL;
2773 encidx = 0;
2774 n = StringValueCStr(*arg);
2775 }
2776 else {
2777 n = rb_enc_name(enc);
2778 }
2779
2780 *name_p = n;
2781 *enc_p = enc;
2782
2783 return encidx;
2784}
2785
2786static int
2787str_transcode_enc_args(VALUE str, VALUE *arg1, VALUE *arg2,
2788 const char **sname_p, rb_encoding **senc_p,
2789 const char **dname_p, rb_encoding **denc_p)
2790{
2791 rb_encoding *senc, *denc;
2792 const char *sname, *dname;
2793 int sencidx, dencidx;
2794
2795 dencidx = enc_arg(arg1, &dname, &denc);
2796
2797 if (NIL_P(*arg2)) {
2798 sencidx = rb_enc_get_index(str);
2799 senc = rb_enc_from_index(sencidx);
2800 sname = rb_enc_name(senc);
2801 }
2802 else {
2803 sencidx = enc_arg(arg2, &sname, &senc);
2804 }
2805
2806 *sname_p = sname;
2807 *senc_p = senc;
2808 *dname_p = dname;
2809 *denc_p = denc;
2810 return dencidx;
2811}
2812
2813static int
2814str_transcode0(int argc, VALUE *argv, VALUE *self, int ecflags, VALUE ecopts)
2815{
2816 VALUE dest;
2817 VALUE str = *self;
2818 VALUE arg1, arg2;
2819 long blen, slen;
2820 unsigned char *buf, *bp, *sp;
2821 const unsigned char *fromp;
2822 rb_encoding *senc, *denc;
2823 const char *sname, *dname;
2824 int dencidx;
2825 int explicitly_invalid_replace = TRUE;
2826
2827 rb_check_arity(argc, 0, 2);
2828
2829 if (argc == 0) {
2830 arg1 = rb_enc_default_internal();
2831 if (NIL_P(arg1)) {
2832 if (!ecflags) return -1;
2833 arg1 = rb_obj_encoding(str);
2834 }
2835 if (!(ecflags & ECONV_INVALID_MASK)) {
2836 explicitly_invalid_replace = FALSE;
2837 }
2839 }
2840 else {
2841 arg1 = argv[0];
2842 }
2843 arg2 = argc<=1 ? Qnil : argv[1];
2844 dencidx = str_transcode_enc_args(str, &arg1, &arg2, &sname, &senc, &dname, &denc);
2845
2846 if ((ecflags & (ECONV_NEWLINE_DECORATOR_MASK|
2850 if (senc && senc == denc) {
2851 if ((ecflags & ECONV_INVALID_MASK) && explicitly_invalid_replace) {
2852 VALUE rep = Qnil;
2853 if (!NIL_P(ecopts)) {
2854 rep = rb_hash_aref(ecopts, sym_replace);
2855 }
2856 dest = rb_enc_str_scrub(senc, str, rep);
2857 if (NIL_P(dest)) dest = str;
2858 *self = dest;
2859 return dencidx;
2860 }
2861 return NIL_P(arg2) ? -1 : dencidx;
2862 }
2863 if (senc && denc && rb_enc_asciicompat(senc) && rb_enc_asciicompat(denc)) {
2864 if (is_ascii_string(str)) {
2865 return dencidx;
2866 }
2867 }
2868 if (encoding_equal(sname, dname)) {
2869 return NIL_P(arg2) ? -1 : dencidx;
2870 }
2871 }
2872 else {
2873 if (senc && denc && !rb_enc_asciicompat(senc) && !rb_enc_asciicompat(denc)) {
2874 rb_encoding *utf8 = rb_utf8_encoding();
2875 str = rb_str_conv_enc(str, senc, utf8);
2876 senc = utf8;
2877 sname = "UTF-8";
2878 }
2879 if (encoding_equal(sname, dname)) {
2880 sname = "";
2881 dname = "";
2882 }
2883 }
2884
2885 fromp = sp = (unsigned char *)RSTRING_PTR(str);
2886 slen = RSTRING_LEN(str);
2887 blen = slen + 30; /* len + margin */
2888 dest = rb_str_tmp_new(blen);
2889 bp = (unsigned char *)RSTRING_PTR(dest);
2890
2891 transcode_loop(&fromp, &bp, (sp+slen), (bp+blen), dest, str_transcoding_resize, sname, dname, ecflags, ecopts, str);
2892 if (fromp != sp+slen) {
2893 rb_raise(rb_eArgError, "not fully converted, %"PRIdPTRDIFF" bytes left", sp+slen-fromp);
2894 }
2895 buf = (unsigned char *)RSTRING_PTR(dest);
2896 *bp = '\0';
2897 rb_str_set_len(dest, bp - buf);
2898
2899 /* set encoding */
2900 if (!denc) {
2901 dencidx = rb_define_dummy_encoding(dname);
2902 RB_GC_GUARD(arg1);
2903 RB_GC_GUARD(arg2);
2904 }
2905 *self = dest;
2906
2907 return dencidx;
2908}
2909
2910static int
2911str_transcode(int argc, VALUE *argv, VALUE *self)
2912{
2913 VALUE opt;
2914 int ecflags = 0;
2915 VALUE ecopts = Qnil;
2916
2917 argc = rb_scan_args(argc, argv, "02:", NULL, NULL, &opt);
2918 if (!NIL_P(opt)) {
2919 ecflags = rb_econv_prepare_opts(opt, &ecopts);
2920 }
2921 return str_transcode0(argc, argv, self, ecflags, ecopts);
2922}
2923
2924static inline VALUE
2925str_encode_associate(VALUE str, int encidx)
2926{
2927 int cr = 0;
2928
2929 rb_enc_associate_index(str, encidx);
2930
2931 /* transcoded string never be broken. */
2932 if (rb_enc_asciicompat(rb_enc_from_index(encidx))) {
2933 rb_str_coderange_scan_restartable(RSTRING_PTR(str), RSTRING_END(str), 0, &cr);
2934 }
2935 else {
2937 }
2938 ENC_CODERANGE_SET(str, cr);
2939 return str;
2940}
2941
2942/*
2943 * call-seq:
2944 * encode!(dst_encoding = Encoding.default_internal, **enc_opts) -> self
2945 * encode!(dst_encoding, src_encoding, **enc_opts) -> self
2946 *
2947 * Like #encode, but applies encoding changes to +self+; returns +self+.
2948 *
2949 * Related: see {Modifying}[rdoc-ref:String@Modifying].
2950 */
2951
2952static VALUE
2953str_encode_bang(int argc, VALUE *argv, VALUE str)
2954{
2955 VALUE newstr;
2956 int encidx;
2957
2958 rb_check_frozen(str);
2959
2960 newstr = str;
2961 encidx = str_transcode(argc, argv, &newstr);
2962
2963 if (encidx < 0) return str;
2964 if (newstr == str) {
2965 rb_enc_associate_index(str, encidx);
2966 return str;
2967 }
2968 rb_str_shared_replace(str, newstr);
2969 return str_encode_associate(str, encidx);
2970}
2971
2972static VALUE encoded_dup(VALUE newstr, VALUE str, int encidx);
2973
2974/*
2975 * call-seq:
2976 * encode(dst_encoding = Encoding.default_internal, **enc_opts) -> string
2977 * encode(dst_encoding, src_encoding, **enc_opts) -> string
2978 *
2979 * :include: doc/string/encode.rdoc
2980 *
2981 */
2982
2983static VALUE
2984str_encode(int argc, VALUE *argv, VALUE str)
2985{
2986 VALUE newstr = str;
2987 int encidx = str_transcode(argc, argv, &newstr);
2988 return encoded_dup(newstr, str, encidx);
2989}
2990
2991VALUE
2992rb_str_encode(VALUE str, VALUE to, int ecflags, VALUE ecopts)
2993{
2994 int argc = 1;
2995 VALUE *argv = &to;
2996 VALUE newstr = str;
2997 int encidx = str_transcode0(argc, argv, &newstr, ecflags, ecopts);
2998 return encoded_dup(newstr, str, encidx);
2999}
3000
3001static VALUE
3002encoded_dup(VALUE newstr, VALUE str, int encidx)
3003{
3004 if (encidx < 0) return rb_str_dup(str);
3005 if (newstr == str) {
3006 newstr = rb_str_dup(str);
3007 rb_enc_associate_index(newstr, encidx);
3008 return newstr;
3009 }
3010 else {
3011 RBASIC_SET_CLASS(newstr, rb_obj_class(str));
3012 }
3013 return str_encode_associate(newstr, encidx);
3014}
3015
3016/*
3017 * Document-class: Encoding::Converter
3018 *
3019 * Encoding conversion class.
3020 */
3021static void
3022econv_free(void *ptr)
3023{
3024 rb_econv_t *ec = ptr;
3025 rb_econv_close(ec);
3026}
3027
3028static size_t
3029econv_memsize(const void *ptr)
3030{
3031 return sizeof(rb_econv_t);
3032}
3033
3034static const rb_data_type_t econv_data_type = {
3035 "econv",
3036 {0, econv_free, econv_memsize,},
3037 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
3038};
3039
3040static VALUE
3041econv_s_allocate(VALUE klass)
3042{
3043 return TypedData_Wrap_Struct(klass, &econv_data_type, NULL);
3044}
3045
3046static rb_encoding *
3047make_dummy_encoding(const char *name)
3048{
3049 rb_encoding *enc;
3050 int idx;
3051 idx = rb_define_dummy_encoding(name);
3052 enc = rb_enc_from_index(idx);
3053 return enc;
3054}
3055
3056static rb_encoding *
3057make_encoding(const char *name)
3058{
3059 rb_encoding *enc;
3060 enc = rb_enc_find(name);
3061 if (!enc) {
3062 RB_VM_LOCKING() {
3063 if (rb_enc_registered(name)) {
3064 enc = NULL;
3065 }
3066 else {
3067 enc = make_dummy_encoding(name);
3068 }
3069 }
3070 }
3071 return enc;
3072}
3073
3074static VALUE
3075make_encobj(const char *name)
3076{
3077 return rb_enc_from_encoding(make_encoding(name));
3078}
3079
3080/*
3081 * call-seq:
3082 * Encoding::Converter.asciicompat_encoding(string) -> encoding or nil
3083 * Encoding::Converter.asciicompat_encoding(encoding) -> encoding or nil
3084 *
3085 * Returns the corresponding ASCII compatible encoding.
3086 *
3087 * Returns nil if the argument is an ASCII compatible encoding.
3088 *
3089 * "corresponding ASCII compatible encoding" is an ASCII compatible encoding which
3090 * can represents exactly the same characters as the given ASCII incompatible encoding.
3091 * So, no conversion undefined error occurs when converting between the two encodings.
3092 *
3093 * Encoding::Converter.asciicompat_encoding("ISO-2022-JP") #=> #<Encoding:stateless-ISO-2022-JP>
3094 * Encoding::Converter.asciicompat_encoding("UTF-16BE") #=> #<Encoding:UTF-8>
3095 * Encoding::Converter.asciicompat_encoding("UTF-8") #=> nil
3096 *
3097 */
3098static VALUE
3099econv_s_asciicompat_encoding(VALUE klass, VALUE arg)
3100{
3101 const char *arg_name, *result_name;
3102 rb_encoding *arg_enc, *result_enc;
3103 VALUE enc = Qnil;
3104
3105 enc_arg(&arg, &arg_name, &arg_enc);
3106 result_name = rb_econv_asciicompat_encoding(arg_name);
3107 if (result_name) {
3108 result_enc = make_encoding(result_name);
3109 enc = rb_enc_from_encoding(result_enc);
3110 }
3111 return enc;
3112}
3113
3114static void
3115econv_args(int argc, VALUE *argv,
3116 VALUE *snamev_p, VALUE *dnamev_p,
3117 const char **sname_p, const char **dname_p,
3118 rb_encoding **senc_p, rb_encoding **denc_p,
3119 int *ecflags_p,
3120 VALUE *ecopts_p)
3121{
3122 VALUE opt, flags_v, ecopts;
3123 int sidx, didx;
3124 const char *sname, *dname;
3125 rb_encoding *senc, *denc;
3126 int ecflags;
3127
3128 argc = rb_scan_args(argc, argv, "21:", snamev_p, dnamev_p, &flags_v, &opt);
3129
3130 if (!NIL_P(flags_v)) {
3131 if (!NIL_P(opt)) {
3132 rb_error_arity(argc + 1, 2, 3);
3133 }
3134 ecflags = NUM2INT(rb_to_int(flags_v));
3135 ecopts = Qnil;
3136 }
3137 else if (!NIL_P(opt)) {
3138 ecflags = rb_econv_prepare_opts(opt, &ecopts);
3139 }
3140 else {
3141 ecflags = 0;
3142 ecopts = Qnil;
3143 }
3144
3145 senc = NULL;
3146 sidx = rb_to_encoding_index(*snamev_p);
3147 if (0 <= sidx) {
3148 senc = rb_enc_from_index(sidx);
3149 }
3150 else {
3151 StringValue(*snamev_p);
3152 }
3153
3154 denc = NULL;
3155 didx = rb_to_encoding_index(*dnamev_p);
3156 if (0 <= didx) {
3157 denc = rb_enc_from_index(didx);
3158 }
3159 else {
3160 StringValue(*dnamev_p);
3161 }
3162
3163 sname = senc ? rb_enc_name(senc) : StringValueCStr(*snamev_p);
3164 dname = denc ? rb_enc_name(denc) : StringValueCStr(*dnamev_p);
3165
3166 *sname_p = sname;
3167 *dname_p = dname;
3168 *senc_p = senc;
3169 *denc_p = denc;
3170 *ecflags_p = ecflags;
3171 *ecopts_p = ecopts;
3172}
3173
3174static int
3175decorate_convpath(VALUE convpath, int ecflags)
3176{
3177 int num_decorators;
3178 const char *decorators[MAX_ECFLAGS_DECORATORS];
3179 int i;
3180 int n, len;
3181
3182 num_decorators = decorator_names(ecflags, decorators);
3183 if (num_decorators == -1)
3184 return -1;
3185
3186 len = n = RARRAY_LENINT(convpath);
3187 if (n != 0) {
3188 VALUE pair = RARRAY_AREF(convpath, n-1);
3189 if (RB_TYPE_P(pair, T_ARRAY)) {
3190 const char *sname = rb_enc_name(rb_to_encoding(RARRAY_AREF(pair, 0)));
3191 const char *dname = rb_enc_name(rb_to_encoding(RARRAY_AREF(pair, 1)));
3192 transcoder_entry_t *entry;
3193 const rb_transcoder *tr;
3194 entry = get_transcoder_entry(sname, dname);
3195 tr = load_transcoder_entry(entry);
3196 if (!tr)
3197 return -1;
3198 if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding) &&
3199 tr->asciicompat_type == asciicompat_encoder) {
3200 n--;
3201 rb_ary_store(convpath, len + num_decorators - 1, pair);
3202 }
3203 }
3204 else {
3205 rb_ary_store(convpath, len + num_decorators - 1, pair);
3206 }
3207 }
3208
3209 for (i = 0; i < num_decorators; i++)
3210 rb_ary_store(convpath, n + i, rb_str_new_cstr(decorators[i]));
3211
3212 return 0;
3213}
3214
3215static void
3216search_convpath_i(const char *sname, const char *dname, int depth, void *arg)
3217{
3218 VALUE *ary_p = arg;
3219 VALUE v;
3220
3221 if (NIL_P(*ary_p)) {
3222 *ary_p = rb_ary_new();
3223 }
3224
3225 if (DECORATOR_P(sname, dname)) {
3226 v = rb_str_new_cstr(dname);
3227 }
3228 else {
3229 v = rb_assoc_new(make_encobj(sname), make_encobj(dname));
3230 }
3231 rb_ary_store(*ary_p, depth, v);
3232}
3233
3234/*
3235 * call-seq:
3236 * Encoding::Converter.search_convpath(source_encoding, destination_encoding) -> ary
3237 * Encoding::Converter.search_convpath(source_encoding, destination_encoding, opt) -> ary
3238 *
3239 * Returns a conversion path.
3240 *
3241 * p Encoding::Converter.search_convpath("ISO-8859-1", "EUC-JP")
3242 * #=> [[#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>],
3243 * # [#<Encoding:UTF-8>, #<Encoding:EUC-JP>]]
3244 *
3245 * p Encoding::Converter.search_convpath("ISO-8859-1", "EUC-JP", universal_newline: true)
3246 * or
3247 * p Encoding::Converter.search_convpath("ISO-8859-1", "EUC-JP", newline: :universal)
3248 * #=> [[#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>],
3249 * # [#<Encoding:UTF-8>, #<Encoding:EUC-JP>],
3250 * # "universal_newline"]
3251 *
3252 * p Encoding::Converter.search_convpath("ISO-8859-1", "UTF-32BE", universal_newline: true)
3253 * or
3254 * p Encoding::Converter.search_convpath("ISO-8859-1", "UTF-32BE", newline: :universal)
3255 * #=> [[#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>],
3256 * # "universal_newline",
3257 * # [#<Encoding:UTF-8>, #<Encoding:UTF-32BE>]]
3258 */
3259static VALUE
3260econv_s_search_convpath(int argc, VALUE *argv, VALUE klass)
3261{
3262 VALUE snamev, dnamev;
3263 const char *sname, *dname;
3264 rb_encoding *senc, *denc;
3265 int ecflags;
3266 VALUE ecopts;
3267 VALUE convpath;
3268
3269 econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
3270
3271 convpath = Qnil;
3272 transcode_search_path(sname, dname, search_convpath_i, &convpath);
3273
3274 if (NIL_P(convpath)) {
3275 VALUE exc = rb_econv_open_exc(sname, dname, ecflags);
3276 RB_GC_GUARD(snamev);
3277 RB_GC_GUARD(dnamev);
3278 rb_exc_raise(exc);
3279 }
3280
3281 if (decorate_convpath(convpath, ecflags) == -1) {
3282 VALUE exc = rb_econv_open_exc(sname, dname, ecflags);
3283 RB_GC_GUARD(snamev);
3284 RB_GC_GUARD(dnamev);
3285 rb_exc_raise(exc);
3286 }
3287
3288 return convpath;
3289}
3290
3291/*
3292 * Check the existence of a conversion path.
3293 * Returns the number of converters in the conversion path.
3294 * result: >=0:success -1:failure
3295 */
3296int
3297rb_econv_has_convpath_p(const char* from_encoding, const char* to_encoding)
3298{
3299 VALUE convpath = Qnil;
3300 transcode_search_path(from_encoding, to_encoding, search_convpath_i,
3301 &convpath);
3302 return RTEST(convpath);
3303}
3304
3306 rb_econv_t *ec;
3307 int index;
3308 int ret;
3309};
3310
3311static void
3312rb_econv_init_by_convpath_i(const char *sname, const char *dname, int depth, void *arg)
3313{
3315 int ret;
3316
3317 if (a->ret == -1)
3318 return;
3319
3320 ret = rb_econv_add_converter(a->ec, sname, dname, a->index);
3321
3322 a->ret = ret;
3323 return;
3324}
3325
3326static rb_econv_t *
3327rb_econv_init_by_convpath(VALUE self, VALUE convpath,
3328 const char **sname_p, const char **dname_p,
3329 rb_encoding **senc_p, rb_encoding**denc_p)
3330{
3331 rb_econv_t *ec;
3332 long i;
3333 int ret, first=1;
3334 VALUE elt;
3335 rb_encoding *senc = 0, *denc = 0;
3336 const char *sname, *dname;
3337
3338 ec = rb_econv_alloc(RARRAY_LENINT(convpath));
3339 DATA_PTR(self) = ec;
3340
3341 for (i = 0; i < RARRAY_LEN(convpath); i++) {
3342 VALUE snamev, dnamev;
3343 VALUE pair;
3344 elt = rb_ary_entry(convpath, i);
3345 if (!NIL_P(pair = rb_check_array_type(elt))) {
3346 if (RARRAY_LEN(pair) != 2)
3347 rb_raise(rb_eArgError, "not a 2-element array in convpath");
3348 snamev = rb_ary_entry(pair, 0);
3349 enc_arg(&snamev, &sname, &senc);
3350 dnamev = rb_ary_entry(pair, 1);
3351 enc_arg(&dnamev, &dname, &denc);
3352 }
3353 else {
3354 sname = "";
3355 dname = StringValueCStr(elt);
3356 }
3357 if (DECORATOR_P(sname, dname)) {
3358 ret = rb_econv_add_converter(ec, sname, dname, ec->num_trans);
3359 if (ret == -1) {
3360 VALUE msg = rb_sprintf("decoration failed: %s", dname);
3361 RB_GC_GUARD(snamev);
3362 RB_GC_GUARD(dnamev);
3363 rb_exc_raise(rb_exc_new_str(rb_eArgError, msg));
3364 }
3365 }
3366 else {
3367 int j = ec->num_trans;
3368 struct rb_econv_init_by_convpath_t arg;
3369 arg.ec = ec;
3370 arg.index = ec->num_trans;
3371 arg.ret = 0;
3372 ret = transcode_search_path(sname, dname, rb_econv_init_by_convpath_i, &arg);
3373 if (ret == -1 || arg.ret == -1) {
3374 VALUE msg = rb_sprintf("adding conversion failed: %s to %s", sname, dname);
3375 RB_GC_GUARD(snamev);
3376 RB_GC_GUARD(dnamev);
3377 rb_exc_raise(rb_exc_new_str(rb_eArgError, msg));
3378 }
3379 if (first) {
3380 first = 0;
3381 *senc_p = senc;
3382 *sname_p = ec->elems[j].tc->transcoder->src_encoding;
3383 }
3384 *denc_p = denc;
3385 *dname_p = ec->elems[ec->num_trans-1].tc->transcoder->dst_encoding;
3386 }
3387 }
3388
3389 if (first) {
3390 *senc_p = NULL;
3391 *denc_p = NULL;
3392 *sname_p = "";
3393 *dname_p = "";
3394 }
3395
3396 ec->source_encoding_name = *sname_p;
3397 ec->destination_encoding_name = *dname_p;
3398
3399 return ec;
3400}
3401
3402/*
3403 * call-seq:
3404 * Encoding::Converter.new(source_encoding, destination_encoding)
3405 * Encoding::Converter.new(source_encoding, destination_encoding, opt)
3406 * Encoding::Converter.new(convpath)
3407 *
3408 * possible options elements:
3409 * hash form:
3410 * :invalid => nil # raise error on invalid byte sequence (default)
3411 * :invalid => :replace # replace invalid byte sequence
3412 * :undef => nil # raise error on undefined conversion (default)
3413 * :undef => :replace # replace undefined conversion
3414 * :replace => string # replacement string ("?" or "\uFFFD" if not specified)
3415 * :newline => :universal # decorator for converting CRLF and CR to LF
3416 * :newline => :lf # decorator for converting CRLF and CR to LF when writing
3417 * :newline => :crlf # decorator for converting LF to CRLF
3418 * :newline => :cr # decorator for converting LF to CR
3419 * :universal_newline => true # decorator for converting CRLF and CR to LF
3420 * :crlf_newline => true # decorator for converting LF to CRLF
3421 * :cr_newline => true # decorator for converting LF to CR
3422 * :lf_newline => true # decorator for converting CRLF and CR to LF when writing
3423 * :xml => :text # escape as XML CharData.
3424 * :xml => :attr # escape as XML AttValue
3425 * integer form:
3426 * Encoding::Converter::INVALID_REPLACE
3427 * Encoding::Converter::UNDEF_REPLACE
3428 * Encoding::Converter::UNDEF_HEX_CHARREF
3429 * Encoding::Converter::UNIVERSAL_NEWLINE_DECORATOR
3430 * Encoding::Converter::LF_NEWLINE_DECORATOR
3431 * Encoding::Converter::CRLF_NEWLINE_DECORATOR
3432 * Encoding::Converter::CR_NEWLINE_DECORATOR
3433 * Encoding::Converter::XML_TEXT_DECORATOR
3434 * Encoding::Converter::XML_ATTR_CONTENT_DECORATOR
3435 * Encoding::Converter::XML_ATTR_QUOTE_DECORATOR
3436 *
3437 * Encoding::Converter.new creates an instance of Encoding::Converter.
3438 *
3439 * Source_encoding and destination_encoding should be a string or
3440 * Encoding object.
3441 *
3442 * opt should be nil, a hash or an integer.
3443 *
3444 * convpath should be an array.
3445 * convpath may contain
3446 * - two-element arrays which contain encodings or encoding names, or
3447 * - strings representing decorator names.
3448 *
3449 * Encoding::Converter.new optionally takes an option.
3450 * The option should be a hash or an integer.
3451 * The option hash can contain :invalid => nil, etc.
3452 * The option integer should be logical-or of constants such as
3453 * Encoding::Converter::INVALID_REPLACE, etc.
3454 *
3455 * [:invalid => nil]
3456 * Raise error on invalid byte sequence. This is a default behavior.
3457 * [:invalid => :replace]
3458 * Replace invalid byte sequence by replacement string.
3459 * [:undef => nil]
3460 * Raise an error if a character in source_encoding is not defined in destination_encoding.
3461 * This is a default behavior.
3462 * [:undef => :replace]
3463 * Replace undefined character in destination_encoding with replacement string.
3464 * [:replace => string]
3465 * Specify the replacement string.
3466 * If not specified, "\uFFFD" is used for Unicode encodings and "?" for others.
3467 * [:universal_newline => true]
3468 * Convert CRLF and CR to LF.
3469 * [:crlf_newline => true]
3470 * Convert LF to CRLF.
3471 * [:cr_newline => true]
3472 * Convert LF to CR.
3473 * [:lf_newline => true]
3474 * Convert CRLF and CR to LF (when writing).
3475 * [:xml => :text]
3476 * Escape as XML CharData.
3477 * This form can be used as an HTML 4.0 #PCDATA.
3478 * - '&' -> '&amp;'
3479 * - '<' -> '&lt;'
3480 * - '>' -> '&gt;'
3481 * - undefined characters in destination_encoding -> hexadecimal CharRef such as &#xHH;
3482 * [:xml => :attr]
3483 * Escape as XML AttValue.
3484 * The converted result is quoted as "...".
3485 * This form can be used as an HTML 4.0 attribute value.
3486 * - '&' -> '&amp;'
3487 * - '<' -> '&lt;'
3488 * - '>' -> '&gt;'
3489 * - '"' -> '&quot;'
3490 * - undefined characters in destination_encoding -> hexadecimal CharRef such as &#xHH;
3491 *
3492 * Examples:
3493 * # UTF-16BE to UTF-8
3494 * ec = Encoding::Converter.new("UTF-16BE", "UTF-8")
3495 *
3496 * # Usually, decorators such as newline conversion are inserted last.
3497 * ec = Encoding::Converter.new("UTF-16BE", "UTF-8", :universal_newline => true)
3498 * p ec.convpath #=> [[#<Encoding:UTF-16BE>, #<Encoding:UTF-8>],
3499 * # "universal_newline"]
3500 *
3501 * # But, if the last encoding is ASCII incompatible,
3502 * # decorators are inserted before the last conversion.
3503 * ec = Encoding::Converter.new("UTF-8", "UTF-16BE", :crlf_newline => true)
3504 * p ec.convpath #=> ["crlf_newline",
3505 * # [#<Encoding:UTF-8>, #<Encoding:UTF-16BE>]]
3506 *
3507 * # Conversion path can be specified directly.
3508 * ec = Encoding::Converter.new(["universal_newline", ["EUC-JP", "UTF-8"], ["UTF-8", "UTF-16BE"]])
3509 * p ec.convpath #=> ["universal_newline",
3510 * # [#<Encoding:EUC-JP>, #<Encoding:UTF-8>],
3511 * # [#<Encoding:UTF-8>, #<Encoding:UTF-16BE>]]
3512 */
3513static VALUE
3514econv_init(int argc, VALUE *argv, VALUE self)
3515{
3516 VALUE ecopts;
3517 VALUE snamev, dnamev;
3518 const char *sname, *dname;
3519 rb_encoding *senc, *denc;
3520 rb_econv_t *ec;
3521 int ecflags;
3522 VALUE convpath;
3523
3524 if (rb_check_typeddata(self, &econv_data_type)) {
3525 rb_raise(rb_eTypeError, "already initialized");
3526 }
3527
3528 if (argc == 1 && !NIL_P(convpath = rb_check_array_type(argv[0]))) {
3529 ec = rb_econv_init_by_convpath(self, convpath, &sname, &dname, &senc, &denc);
3530 ecflags = 0;
3531 ecopts = Qnil;
3532 }
3533 else {
3534 econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
3535 ec = rb_econv_open_opts(sname, dname, ecflags, ecopts);
3536 }
3537
3538 if (!ec) {
3539 VALUE exc = rb_econv_open_exc(sname, dname, ecflags);
3540 RB_GC_GUARD(snamev);
3541 RB_GC_GUARD(dnamev);
3542 rb_exc_raise(exc);
3543 }
3544
3545 if (!DECORATOR_P(sname, dname)) {
3546 if (!senc)
3547 senc = make_dummy_encoding(sname);
3548 if (!denc)
3549 denc = make_dummy_encoding(dname);
3550 RB_GC_GUARD(snamev);
3551 RB_GC_GUARD(dnamev);
3552 }
3553
3554 ec->source_encoding = senc;
3555 ec->destination_encoding = denc;
3556
3557 DATA_PTR(self) = ec;
3558
3559 return self;
3560}
3561
3562/*
3563 * call-seq:
3564 * ec.inspect -> string
3565 *
3566 * Returns a printable version of <i>ec</i>
3567 *
3568 * ec = Encoding::Converter.new("iso-8859-1", "utf-8")
3569 * puts ec.inspect #=> #<Encoding::Converter: ISO-8859-1 to UTF-8>
3570 *
3571 */
3572static VALUE
3573econv_inspect(VALUE self)
3574{
3575 const char *cname = rb_obj_classname(self);
3576 rb_econv_t *ec;
3577
3578 TypedData_Get_Struct(self, rb_econv_t, &econv_data_type, ec);
3579 if (!ec)
3580 return rb_sprintf("#<%s: uninitialized>", cname);
3581 else {
3582 const char *sname = ec->source_encoding_name;
3583 const char *dname = ec->destination_encoding_name;
3584 VALUE str;
3585 str = rb_sprintf("#<%s: ", cname);
3586 econv_description(sname, dname, ec->flags, str);
3587 rb_str_cat2(str, ">");
3588 return str;
3589 }
3590}
3591
3592static rb_econv_t *
3593check_econv(VALUE self)
3594{
3595 rb_econv_t *ec;
3596
3597 TypedData_Get_Struct(self, rb_econv_t, &econv_data_type, ec);
3598 if (!ec) {
3599 rb_raise(rb_eTypeError, "uninitialized encoding converter");
3600 }
3601 return ec;
3602}
3603
3604static VALUE
3605econv_get_encoding(rb_encoding *encoding)
3606{
3607 if (!encoding)
3608 return Qnil;
3609 return rb_enc_from_encoding(encoding);
3610}
3611
3612/*
3613 * call-seq:
3614 * ec.source_encoding -> encoding
3615 *
3616 * Returns the source encoding as an Encoding object.
3617 */
3618static VALUE
3619econv_source_encoding(VALUE self)
3620{
3621 rb_econv_t *ec = check_econv(self);
3622 return econv_get_encoding(ec->source_encoding);
3623}
3624
3625/*
3626 * call-seq:
3627 * ec.destination_encoding -> encoding
3628 *
3629 * Returns the destination encoding as an Encoding object.
3630 */
3631static VALUE
3632econv_destination_encoding(VALUE self)
3633{
3634 rb_econv_t *ec = check_econv(self);
3635 return econv_get_encoding(ec->destination_encoding);
3636}
3637
3638/*
3639 * call-seq:
3640 * ec.convpath -> ary
3641 *
3642 * Returns the conversion path of ec.
3643 *
3644 * The result is an array of conversions.
3645 *
3646 * ec = Encoding::Converter.new("ISO-8859-1", "EUC-JP", crlf_newline: true)
3647 * p ec.convpath
3648 * #=> [[#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>],
3649 * # [#<Encoding:UTF-8>, #<Encoding:EUC-JP>],
3650 * # "crlf_newline"]
3651 *
3652 * Each element of the array is a pair of encodings or a string.
3653 * A pair means an encoding conversion.
3654 * A string means a decorator.
3655 *
3656 * In the above example, [#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>] means
3657 * a converter from ISO-8859-1 to UTF-8.
3658 * "crlf_newline" means newline converter from LF to CRLF.
3659 */
3660static VALUE
3661econv_convpath(VALUE self)
3662{
3663 rb_econv_t *ec = check_econv(self);
3664 VALUE result;
3665 int i;
3666
3667 result = rb_ary_new();
3668 for (i = 0; i < ec->num_trans; i++) {
3669 const rb_transcoder *tr = ec->elems[i].tc->transcoder;
3670 VALUE v;
3671 if (DECORATOR_P(tr->src_encoding, tr->dst_encoding))
3672 v = rb_str_new_cstr(tr->dst_encoding);
3673 else
3674 v = rb_assoc_new(make_encobj(tr->src_encoding), make_encobj(tr->dst_encoding));
3675 rb_ary_push(result, v);
3676 }
3677 return result;
3678}
3679
3680/*
3681 * call-seq:
3682 * ec == other -> true or false
3683 */
3684static VALUE
3685econv_equal(VALUE self, VALUE other)
3686{
3687 rb_econv_t *ec1 = check_econv(self);
3688 rb_econv_t *ec2;
3689 int i;
3690
3691 if (!rb_typeddata_is_kind_of(other, &econv_data_type)) {
3692 return Qnil;
3693 }
3694 ec2 = DATA_PTR(other);
3695 if (!ec2) return Qfalse;
3696 if (ec1->source_encoding_name != ec2->source_encoding_name &&
3697 strcmp(ec1->source_encoding_name, ec2->source_encoding_name))
3698 return Qfalse;
3699 if (ec1->destination_encoding_name != ec2->destination_encoding_name &&
3700 strcmp(ec1->destination_encoding_name, ec2->destination_encoding_name))
3701 return Qfalse;
3702 if (ec1->flags != ec2->flags) return Qfalse;
3703 if (ec1->replacement_enc != ec2->replacement_enc &&
3704 strcmp(ec1->replacement_enc, ec2->replacement_enc))
3705 return Qfalse;
3706 if (ec1->replacement_len != ec2->replacement_len) return Qfalse;
3707 if (ec1->replacement_str != ec2->replacement_str &&
3708 memcmp(ec1->replacement_str, ec2->replacement_str, ec2->replacement_len))
3709 return Qfalse;
3710
3711 if (ec1->num_trans != ec2->num_trans) return Qfalse;
3712 for (i = 0; i < ec1->num_trans; i++) {
3713 if (ec1->elems[i].tc->transcoder != ec2->elems[i].tc->transcoder)
3714 return Qfalse;
3715 }
3716 return Qtrue;
3717}
3718
3719static VALUE
3720econv_result_to_symbol(rb_econv_result_t res)
3721{
3722 switch (res) {
3723 case econv_invalid_byte_sequence: return sym_invalid_byte_sequence;
3724 case econv_incomplete_input: return sym_incomplete_input;
3725 case econv_undefined_conversion: return sym_undefined_conversion;
3726 case econv_destination_buffer_full: return sym_destination_buffer_full;
3727 case econv_source_buffer_empty: return sym_source_buffer_empty;
3728 case econv_finished: return sym_finished;
3729 case econv_after_output: return sym_after_output;
3730 default: return INT2NUM(res); /* should not be reached */
3731 }
3732}
3733
3734/*
3735 * call-seq:
3736 * ec.primitive_convert(source_buffer, destination_buffer) -> symbol
3737 * ec.primitive_convert(source_buffer, destination_buffer, destination_byteoffset) -> symbol
3738 * ec.primitive_convert(source_buffer, destination_buffer, destination_byteoffset, destination_bytesize) -> symbol
3739 * ec.primitive_convert(source_buffer, destination_buffer, destination_byteoffset, destination_bytesize, opt) -> symbol
3740 *
3741 * possible opt elements:
3742 * hash form:
3743 * :partial_input => true # source buffer may be part of larger source
3744 * :after_output => true # stop conversion after output before input
3745 * integer form:
3746 * Encoding::Converter::PARTIAL_INPUT
3747 * Encoding::Converter::AFTER_OUTPUT
3748 *
3749 * possible results:
3750 * :invalid_byte_sequence
3751 * :incomplete_input
3752 * :undefined_conversion
3753 * :after_output
3754 * :destination_buffer_full
3755 * :source_buffer_empty
3756 * :finished
3757 *
3758 * primitive_convert converts source_buffer into destination_buffer.
3759 *
3760 * source_buffer should be a string or nil.
3761 * nil means an empty string.
3762 *
3763 * destination_buffer should be a string.
3764 *
3765 * destination_byteoffset should be an integer or nil.
3766 * nil means the end of destination_buffer.
3767 * If it is omitted, nil is assumed.
3768 *
3769 * destination_bytesize should be an integer or nil.
3770 * nil means unlimited.
3771 * If it is omitted, nil is assumed.
3772 *
3773 * opt should be nil, a hash or an integer.
3774 * nil means no flags.
3775 * If it is omitted, nil is assumed.
3776 *
3777 * primitive_convert converts the content of source_buffer from beginning
3778 * and store the result into destination_buffer.
3779 *
3780 * destination_byteoffset and destination_bytesize specify the region which
3781 * the converted result is stored.
3782 * destination_byteoffset specifies the start position in destination_buffer in bytes.
3783 * If destination_byteoffset is nil,
3784 * destination_buffer.bytesize is used for appending the result.
3785 * destination_bytesize specifies maximum number of bytes.
3786 * If destination_bytesize is nil,
3787 * destination size is unlimited.
3788 * After conversion, destination_buffer is resized to
3789 * destination_byteoffset + actually produced number of bytes.
3790 * Also destination_buffer's encoding is set to destination_encoding.
3791 *
3792 * primitive_convert drops the converted part of source_buffer.
3793 * the dropped part is converted in destination_buffer or
3794 * buffered in Encoding::Converter object.
3795 *
3796 * primitive_convert stops conversion when one of following condition met.
3797 * - invalid byte sequence found in source buffer (:invalid_byte_sequence)
3798 * +primitive_errinfo+ and +last_error+ methods returns the detail of the error.
3799 * - unexpected end of source buffer (:incomplete_input)
3800 * this occur only when :partial_input is not specified.
3801 * +primitive_errinfo+ and +last_error+ methods returns the detail of the error.
3802 * - character not representable in output encoding (:undefined_conversion)
3803 * +primitive_errinfo+ and +last_error+ methods returns the detail of the error.
3804 * - after some output is generated, before input is done (:after_output)
3805 * this occur only when :after_output is specified.
3806 * - destination buffer is full (:destination_buffer_full)
3807 * this occur only when destination_bytesize is non-nil.
3808 * - source buffer is empty (:source_buffer_empty)
3809 * this occur only when :partial_input is specified.
3810 * - conversion is finished (:finished)
3811 *
3812 * example:
3813 * ec = Encoding::Converter.new("UTF-8", "UTF-16BE")
3814 * ret = ec.primitive_convert(src="pi", dst="", nil, 100)
3815 * p [ret, src, dst] #=> [:finished, "", "\x00p\x00i"]
3816 *
3817 * ec = Encoding::Converter.new("UTF-8", "UTF-16BE")
3818 * ret = ec.primitive_convert(src="pi", dst="", nil, 1)
3819 * p [ret, src, dst] #=> [:destination_buffer_full, "i", "\x00"]
3820 * ret = ec.primitive_convert(src, dst="", nil, 1)
3821 * p [ret, src, dst] #=> [:destination_buffer_full, "", "p"]
3822 * ret = ec.primitive_convert(src, dst="", nil, 1)
3823 * p [ret, src, dst] #=> [:destination_buffer_full, "", "\x00"]
3824 * ret = ec.primitive_convert(src, dst="", nil, 1)
3825 * p [ret, src, dst] #=> [:finished, "", "i"]
3826 *
3827 */
3828static VALUE
3829econv_primitive_convert(int argc, VALUE *argv, VALUE self)
3830{
3831 VALUE input, output, output_byteoffset_v, output_bytesize_v, opt, flags_v;
3832 rb_econv_t *ec = check_econv(self);
3834 const unsigned char *ip, *is;
3835 unsigned char *op, *os;
3836 long output_byteoffset, output_bytesize;
3837 unsigned long output_byteend;
3838 int flags;
3839
3840 argc = rb_scan_args(argc, argv, "23:", &input, &output, &output_byteoffset_v, &output_bytesize_v, &flags_v, &opt);
3841
3842 if (NIL_P(output_byteoffset_v))
3843 output_byteoffset = 0; /* dummy */
3844 else
3845 output_byteoffset = NUM2LONG(output_byteoffset_v);
3846
3847 if (NIL_P(output_bytesize_v))
3848 output_bytesize = 0; /* dummy */
3849 else
3850 output_bytesize = NUM2LONG(output_bytesize_v);
3851
3852 if (!NIL_P(flags_v)) {
3853 if (!NIL_P(opt)) {
3854 rb_error_arity(argc + 1, 2, 5);
3855 }
3856 flags = NUM2INT(rb_to_int(flags_v));
3857 }
3858 else if (!NIL_P(opt)) {
3859 VALUE v;
3860 flags = 0;
3861 v = rb_hash_aref(opt, sym_partial_input);
3862 if (RTEST(v))
3863 flags |= ECONV_PARTIAL_INPUT;
3864 v = rb_hash_aref(opt, sym_after_output);
3865 if (RTEST(v))
3866 flags |= ECONV_AFTER_OUTPUT;
3867 }
3868 else {
3869 flags = 0;
3870 }
3871
3872 StringValue(output);
3873 if (!NIL_P(input))
3874 StringValue(input);
3875 rb_str_modify(output);
3876 if (input == output) {
3877 rb_raise(rb_eArgError, "cannot convert into the source string");
3878 }
3879
3880 if (NIL_P(output_bytesize_v)) {
3881 output_bytesize = rb_str_capacity(output);
3882
3883 if (!NIL_P(input) && output_bytesize < RSTRING_LEN(input))
3884 output_bytesize = RSTRING_LEN(input);
3885 }
3886
3887 retry:
3888
3889 if (NIL_P(output_byteoffset_v))
3890 output_byteoffset = RSTRING_LEN(output);
3891
3892 if (output_byteoffset < 0)
3893 rb_raise(rb_eArgError, "negative output_byteoffset");
3894
3895 if (RSTRING_LEN(output) < output_byteoffset)
3896 rb_raise(rb_eArgError, "output_byteoffset too big");
3897
3898 if (output_bytesize < 0)
3899 rb_raise(rb_eArgError, "negative output_bytesize");
3900
3901 output_byteend = (unsigned long)output_byteoffset +
3902 (unsigned long)output_bytesize;
3903
3904 if (output_byteend < (unsigned long)output_byteoffset ||
3905 LONG_MAX < output_byteend)
3906 rb_raise(rb_eArgError, "output_byteoffset+output_bytesize too big");
3907
3908 if (rb_str_capacity(output) < output_byteend)
3909 rb_str_resize(output, output_byteend);
3910
3911 if (NIL_P(input)) {
3912 ip = is = NULL;
3913 }
3914 else {
3915 ip = (const unsigned char *)RSTRING_PTR(input);
3916 is = ip + RSTRING_LEN(input);
3917 }
3918
3919 op = (unsigned char *)RSTRING_PTR(output) + output_byteoffset;
3920 os = op + output_bytesize;
3921
3922 res = rb_econv_convert(ec, &ip, is, &op, os, flags);
3923 rb_str_set_len(output, op-(unsigned char *)RSTRING_PTR(output));
3924 if (!NIL_P(input)) {
3925 rb_str_drop_bytes(input, ip - (unsigned char *)RSTRING_PTR(input));
3926 }
3927
3928 if (NIL_P(output_bytesize_v) && res == econv_destination_buffer_full) {
3929 if (LONG_MAX / 2 < output_bytesize)
3930 rb_raise(rb_eArgError, "too long conversion result");
3931 output_bytesize *= 2;
3932 output_byteoffset_v = Qnil;
3933 goto retry;
3934 }
3935
3936 if (ec->destination_encoding) {
3937 rb_enc_associate(output, ec->destination_encoding);
3938 }
3939
3940 return econv_result_to_symbol(res);
3941}
3942
3943/*
3944 * call-seq:
3945 * ec.convert(source_string) -> destination_string
3946 *
3947 * Convert source_string and return destination_string.
3948 *
3949 * source_string is assumed as a part of source.
3950 * i.e. :partial_input=>true is specified internally.
3951 * finish method should be used last.
3952 *
3953 * ec = Encoding::Converter.new("utf-8", "euc-jp")
3954 * puts ec.convert("\u3042").dump #=> "\xA4\xA2"
3955 * puts ec.finish.dump #=> ""
3956 *
3957 * ec = Encoding::Converter.new("euc-jp", "utf-8")
3958 * puts ec.convert("\xA4").dump #=> ""
3959 * puts ec.convert("\xA2").dump #=> "\xE3\x81\x82"
3960 * puts ec.finish.dump #=> ""
3961 *
3962 * ec = Encoding::Converter.new("utf-8", "iso-2022-jp")
3963 * puts ec.convert("\xE3").dump #=> "".force_encoding("ISO-2022-JP")
3964 * puts ec.convert("\x81").dump #=> "".force_encoding("ISO-2022-JP")
3965 * puts ec.convert("\x82").dump #=> "\e$B$\"".force_encoding("ISO-2022-JP")
3966 * puts ec.finish.dump #=> "\e(B".force_encoding("ISO-2022-JP")
3967 *
3968 * If a conversion error occur,
3969 * Encoding::UndefinedConversionError or
3970 * Encoding::InvalidByteSequenceError is raised.
3971 * Encoding::Converter#convert doesn't supply methods to recover or restart
3972 * from these exceptions.
3973 * When you want to handle these conversion errors,
3974 * use Encoding::Converter#primitive_convert.
3975 *
3976 */
3977static VALUE
3978econv_convert(VALUE self, VALUE source_string)
3979{
3980 VALUE ret, dst;
3981 VALUE av[5];
3982 int ac;
3983 rb_econv_t *ec = check_econv(self);
3984
3985 StringValue(source_string);
3986
3987 dst = rb_str_new(NULL, 0);
3988
3989 av[0] = rb_str_dup(source_string);
3990 av[1] = dst;
3991 av[2] = Qnil;
3992 av[3] = Qnil;
3994 ac = 5;
3995
3996 ret = econv_primitive_convert(ac, av, self);
3997
3998 if (ret == sym_invalid_byte_sequence ||
3999 ret == sym_undefined_conversion ||
4000 ret == sym_incomplete_input) {
4001 VALUE exc = make_econv_exception(ec);
4002 rb_exc_raise(exc);
4003 }
4004
4005 if (ret == sym_finished) {
4006 rb_raise(rb_eArgError, "converter already finished");
4007 }
4008
4009 if (ret != sym_source_buffer_empty) {
4010 rb_bug("unexpected result of econv_primitive_convert");
4011 }
4012
4013 return dst;
4014}
4015
4016/*
4017 * call-seq:
4018 * ec.finish -> string
4019 *
4020 * Finishes the converter.
4021 * It returns the last part of the converted string.
4022 *
4023 * ec = Encoding::Converter.new("utf-8", "iso-2022-jp")
4024 * p ec.convert("\u3042") #=> "\e$B$\""
4025 * p ec.finish #=> "\e(B"
4026 */
4027static VALUE
4028econv_finish(VALUE self)
4029{
4030 VALUE ret, dst;
4031 VALUE av[5];
4032 int ac;
4033 rb_econv_t *ec = check_econv(self);
4034
4035 dst = rb_str_new(NULL, 0);
4036
4037 av[0] = Qnil;
4038 av[1] = dst;
4039 av[2] = Qnil;
4040 av[3] = Qnil;
4041 av[4] = INT2FIX(0);
4042 ac = 5;
4043
4044 ret = econv_primitive_convert(ac, av, self);
4045
4046 if (ret == sym_invalid_byte_sequence ||
4047 ret == sym_undefined_conversion ||
4048 ret == sym_incomplete_input) {
4049 VALUE exc = make_econv_exception(ec);
4050 rb_exc_raise(exc);
4051 }
4052
4053 if (ret != sym_finished) {
4054 rb_bug("unexpected result of econv_primitive_convert");
4055 }
4056
4057 return dst;
4058}
4059
4060/*
4061 * call-seq:
4062 * ec.primitive_errinfo -> array
4063 *
4064 * primitive_errinfo returns important information regarding the last error
4065 * as a 5-element array:
4066 *
4067 * [result, enc1, enc2, error_bytes, readagain_bytes]
4068 *
4069 * result is the last result of primitive_convert.
4070 *
4071 * Other elements are only meaningful when result is
4072 * :invalid_byte_sequence, :incomplete_input or :undefined_conversion.
4073 *
4074 * enc1 and enc2 indicate a conversion step as a pair of strings.
4075 * For example, a converter from EUC-JP to ISO-8859-1 converts
4076 * a string as follows: EUC-JP -> UTF-8 -> ISO-8859-1.
4077 * So [enc1, enc2] is either ["EUC-JP", "UTF-8"] or ["UTF-8", "ISO-8859-1"].
4078 *
4079 * error_bytes and readagain_bytes indicate the byte sequences which caused the error.
4080 * error_bytes is discarded portion.
4081 * readagain_bytes is buffered portion which is read again on next conversion.
4082 *
4083 * Example:
4084 *
4085 * # \xff is invalid as EUC-JP.
4086 * ec = Encoding::Converter.new("EUC-JP", "Shift_JIS")
4087 * ec.primitive_convert(src="\xff", dst="", nil, 10)
4088 * p ec.primitive_errinfo
4089 * #=> [:invalid_byte_sequence, "EUC-JP", "Shift_JIS", "\xFF", ""]
4090 *
4091 * # HIRAGANA LETTER A (\xa4\xa2 in EUC-JP) is not representable in ISO-8859-1.
4092 * # Since this error is occur in UTF-8 to ISO-8859-1 conversion,
4093 * # error_bytes is HIRAGANA LETTER A in UTF-8 (\xE3\x81\x82).
4094 * ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1")
4095 * ec.primitive_convert(src="\xa4\xa2", dst="", nil, 10)
4096 * p ec.primitive_errinfo
4097 * #=> [:undefined_conversion, "UTF-8", "ISO-8859-1", "\xE3\x81\x82", ""]
4098 *
4099 * # partial character is invalid
4100 * ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1")
4101 * ec.primitive_convert(src="\xa4", dst="", nil, 10)
4102 * p ec.primitive_errinfo
4103 * #=> [:incomplete_input, "EUC-JP", "UTF-8", "\xA4", ""]
4104 *
4105 * # Encoding::Converter::PARTIAL_INPUT prevents invalid errors by
4106 * # partial characters.
4107 * ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1")
4108 * ec.primitive_convert(src="\xa4", dst="", nil, 10, Encoding::Converter::PARTIAL_INPUT)
4109 * p ec.primitive_errinfo
4110 * #=> [:source_buffer_empty, nil, nil, nil, nil]
4111 *
4112 * # \xd8\x00\x00@ is invalid as UTF-16BE because
4113 * # no low surrogate after high surrogate (\xd8\x00).
4114 * # It is detected by 3rd byte (\00) which is part of next character.
4115 * # So the high surrogate (\xd8\x00) is discarded and
4116 * # the 3rd byte is read again later.
4117 * # Since the byte is buffered in ec, it is dropped from src.
4118 * ec = Encoding::Converter.new("UTF-16BE", "UTF-8")
4119 * ec.primitive_convert(src="\xd8\x00\x00@", dst="", nil, 10)
4120 * p ec.primitive_errinfo
4121 * #=> [:invalid_byte_sequence, "UTF-16BE", "UTF-8", "\xD8\x00", "\x00"]
4122 * p src
4123 * #=> "@"
4124 *
4125 * # Similar to UTF-16BE, \x00\xd8@\x00 is invalid as UTF-16LE.
4126 * # The problem is detected by 4th byte.
4127 * ec = Encoding::Converter.new("UTF-16LE", "UTF-8")
4128 * ec.primitive_convert(src="\x00\xd8@\x00", dst="", nil, 10)
4129 * p ec.primitive_errinfo
4130 * #=> [:invalid_byte_sequence, "UTF-16LE", "UTF-8", "\x00\xD8", "@\x00"]
4131 * p src
4132 * #=> ""
4133 *
4134 */
4135static VALUE
4136econv_primitive_errinfo(VALUE self)
4137{
4138 rb_econv_t *ec = check_econv(self);
4139
4140 VALUE ary;
4141
4142 ary = rb_ary_new2(5);
4143
4144 rb_ary_store(ary, 0, econv_result_to_symbol(ec->last_error.result));
4145 rb_ary_store(ary, 4, Qnil);
4146
4147 if (ec->last_error.source_encoding)
4148 rb_ary_store(ary, 1, rb_str_new2(ec->last_error.source_encoding));
4149
4150 if (ec->last_error.destination_encoding)
4151 rb_ary_store(ary, 2, rb_str_new2(ec->last_error.destination_encoding));
4152
4153 if (ec->last_error.error_bytes_start) {
4154 rb_ary_store(ary, 3, rb_str_new((const char *)ec->last_error.error_bytes_start, ec->last_error.error_bytes_len));
4155 rb_ary_store(ary, 4, rb_str_new((const char *)ec->last_error.error_bytes_start + ec->last_error.error_bytes_len, ec->last_error.readagain_len));
4156 }
4157
4158 return ary;
4159}
4160
4161/*
4162 * call-seq:
4163 * ec.insert_output(string) -> nil
4164 *
4165 * Inserts string into the encoding converter.
4166 * The string will be converted to the destination encoding and
4167 * output on later conversions.
4168 *
4169 * If the destination encoding is stateful,
4170 * string is converted according to the state and the state is updated.
4171 *
4172 * This method should be used only when a conversion error occurs.
4173 *
4174 * ec = Encoding::Converter.new("utf-8", "iso-8859-1")
4175 * src = "HIRAGANA LETTER A is \u{3042}."
4176 * dst = ""
4177 * p ec.primitive_convert(src, dst) #=> :undefined_conversion
4178 * puts "[#{dst.dump}, #{src.dump}]" #=> ["HIRAGANA LETTER A is ", "."]
4179 * ec.insert_output("<err>")
4180 * p ec.primitive_convert(src, dst) #=> :finished
4181 * puts "[#{dst.dump}, #{src.dump}]" #=> ["HIRAGANA LETTER A is <err>.", ""]
4182 *
4183 * ec = Encoding::Converter.new("utf-8", "iso-2022-jp")
4184 * src = "\u{306F 3041 3068 2661 3002}" # U+2661 is not representable in iso-2022-jp
4185 * dst = ""
4186 * p ec.primitive_convert(src, dst) #=> :undefined_conversion
4187 * puts "[#{dst.dump}, #{src.dump}]" #=> ["\e$B$O$!$H".force_encoding("ISO-2022-JP"), "\xE3\x80\x82"]
4188 * ec.insert_output "?" # state change required to output "?".
4189 * p ec.primitive_convert(src, dst) #=> :finished
4190 * puts "[#{dst.dump}, #{src.dump}]" #=> ["\e$B$O$!$H\e(B?\e$B!#\e(B".force_encoding("ISO-2022-JP"), ""]
4191 *
4192 */
4193static VALUE
4194econv_insert_output(VALUE self, VALUE string)
4195{
4196 const char *insert_enc;
4197
4198 int ret;
4199
4200 rb_econv_t *ec = check_econv(self);
4201
4202 StringValue(string);
4203 insert_enc = rb_econv_encoding_to_insert_output(ec);
4204 string = rb_str_encode(string, rb_enc_from_encoding(rb_enc_find(insert_enc)), 0, Qnil);
4205
4206 ret = rb_econv_insert_output(ec, (const unsigned char *)RSTRING_PTR(string), RSTRING_LEN(string), insert_enc);
4207 if (ret == -1) {
4208 rb_raise(rb_eArgError, "too big string");
4209 }
4210
4211 return Qnil;
4212}
4213
4214/*
4215 * call-seq:
4216 * ec.putback -> string
4217 * ec.putback(max_numbytes) -> string
4218 *
4219 * Put back the bytes which will be converted.
4220 *
4221 * The bytes are caused by invalid_byte_sequence error.
4222 * When invalid_byte_sequence error, some bytes are discarded and
4223 * some bytes are buffered to be converted later.
4224 * The latter bytes can be put back.
4225 * It can be observed by
4226 * Encoding::InvalidByteSequenceError#readagain_bytes and
4227 * Encoding::Converter#primitive_errinfo.
4228 *
4229 * ec = Encoding::Converter.new("utf-16le", "iso-8859-1")
4230 * src = "\x00\xd8\x61\x00"
4231 * dst = ""
4232 * p ec.primitive_convert(src, dst) #=> :invalid_byte_sequence
4233 * p ec.primitive_errinfo #=> [:invalid_byte_sequence, "UTF-16LE", "UTF-8", "\x00\xD8", "a\x00"]
4234 * p ec.putback #=> "a\x00"
4235 * p ec.putback #=> "" # no more bytes to put back
4236 *
4237 */
4238static VALUE
4239econv_putback(int argc, VALUE *argv, VALUE self)
4240{
4241 rb_econv_t *ec = check_econv(self);
4242 int n;
4243 int putbackable;
4244 VALUE str, max;
4245
4246 if (!rb_check_arity(argc, 0, 1) || NIL_P(max = argv[0])) {
4247 n = rb_econv_putbackable(ec);
4248 }
4249 else {
4250 n = NUM2INT(max);
4251 putbackable = rb_econv_putbackable(ec);
4252 if (putbackable < n)
4253 n = putbackable;
4254 }
4255
4256 str = rb_str_new(NULL, n);
4257 rb_econv_putback(ec, (unsigned char *)RSTRING_PTR(str), n);
4258
4259 if (ec->source_encoding) {
4260 rb_enc_associate(str, ec->source_encoding);
4261 }
4262
4263 return str;
4264}
4265
4266/*
4267 * call-seq:
4268 * ec.last_error -> exception or nil
4269 *
4270 * Returns an exception object for the last conversion.
4271 * Returns nil if the last conversion did not produce an error.
4272 *
4273 * "error" means that
4274 * Encoding::InvalidByteSequenceError and Encoding::UndefinedConversionError for
4275 * Encoding::Converter#convert and
4276 * :invalid_byte_sequence, :incomplete_input and :undefined_conversion for
4277 * Encoding::Converter#primitive_convert.
4278 *
4279 * ec = Encoding::Converter.new("utf-8", "iso-8859-1")
4280 * p ec.primitive_convert(src="\xf1abcd", dst="") #=> :invalid_byte_sequence
4281 * p ec.last_error #=> #<Encoding::InvalidByteSequenceError: "\xF1" followed by "a" on UTF-8>
4282 * p ec.primitive_convert(src, dst, nil, 1) #=> :destination_buffer_full
4283 * p ec.last_error #=> nil
4284 *
4285 */
4286static VALUE
4287econv_last_error(VALUE self)
4288{
4289 rb_econv_t *ec = check_econv(self);
4290 VALUE exc;
4291
4292 exc = make_econv_exception(ec);
4293 if (NIL_P(exc))
4294 return Qnil;
4295 return exc;
4296}
4297
4298/*
4299 * call-seq:
4300 * ec.replacement -> string
4301 *
4302 * Returns the replacement string.
4303 *
4304 * ec = Encoding::Converter.new("euc-jp", "us-ascii")
4305 * p ec.replacement #=> "?"
4306 *
4307 * ec = Encoding::Converter.new("euc-jp", "utf-8")
4308 * p ec.replacement #=> "\uFFFD"
4309 */
4310static VALUE
4311econv_get_replacement(VALUE self)
4312{
4313 rb_econv_t *ec = check_econv(self);
4314 int ret;
4315 rb_encoding *enc;
4316
4317 ret = make_replacement(ec);
4318 if (ret == -1) {
4319 rb_raise(rb_eUndefinedConversionError, "replacement character setup failed");
4320 }
4321
4322 enc = rb_enc_find(ec->replacement_enc);
4323 return rb_enc_str_new((const char *)ec->replacement_str, (long)ec->replacement_len, enc);
4324}
4325
4326/*
4327 * call-seq:
4328 * ec.replacement = string
4329 *
4330 * Sets the replacement string.
4331 *
4332 * ec = Encoding::Converter.new("utf-8", "us-ascii", :undef => :replace)
4333 * ec.replacement = "<undef>"
4334 * p ec.convert("a \u3042 b") #=> "a <undef> b"
4335 */
4336static VALUE
4337econv_set_replacement(VALUE self, VALUE arg)
4338{
4339 rb_econv_t *ec = check_econv(self);
4340 VALUE string = arg;
4341 int ret;
4342 rb_encoding *enc;
4343
4344 StringValue(string);
4345 enc = rb_enc_get(string);
4346
4347 ret = rb_econv_set_replacement(ec,
4348 (const unsigned char *)RSTRING_PTR(string),
4349 RSTRING_LEN(string),
4350 rb_enc_name(enc));
4351
4352 if (ret == -1) {
4353 /* xxx: rb_eInvalidByteSequenceError? */
4354 rb_raise(rb_eUndefinedConversionError, "replacement character setup failed");
4355 }
4356
4357 return arg;
4358}
4359
4360VALUE
4362{
4363 return make_econv_exception(ec);
4364}
4365
4366void
4368{
4369 VALUE exc;
4370
4371 exc = make_econv_exception(ec);
4372 if (NIL_P(exc))
4373 return;
4374 rb_exc_raise(exc);
4375}
4376
4377/*
4378 * call-seq:
4379 * ecerr.source_encoding_name -> string
4380 *
4381 * Returns the source encoding name as a string.
4382 */
4383static VALUE
4384ecerr_source_encoding_name(VALUE self)
4385{
4386 return rb_attr_get(self, id_source_encoding_name);
4387}
4388
4389/*
4390 * call-seq:
4391 * ecerr.source_encoding -> encoding
4392 *
4393 * Returns the source encoding as an encoding object.
4394 *
4395 * Note that the result may not be equal to the source encoding of
4396 * the encoding converter if the conversion has multiple steps.
4397 *
4398 * ec = Encoding::Converter.new("ISO-8859-1", "EUC-JP") # ISO-8859-1 -> UTF-8 -> EUC-JP
4399 * begin
4400 * ec.convert("\xa0") # NO-BREAK SPACE, which is available in UTF-8 but not in EUC-JP.
4401 * rescue Encoding::UndefinedConversionError
4402 * p $!.source_encoding #=> #<Encoding:UTF-8>
4403 * p $!.destination_encoding #=> #<Encoding:EUC-JP>
4404 * p $!.source_encoding_name #=> "UTF-8"
4405 * p $!.destination_encoding_name #=> "EUC-JP"
4406 * end
4407 *
4408 */
4409static VALUE
4410ecerr_source_encoding(VALUE self)
4411{
4412 return rb_attr_get(self, id_source_encoding);
4413}
4414
4415/*
4416 * call-seq:
4417 * ecerr.destination_encoding_name -> string
4418 *
4419 * Returns the destination encoding name as a string.
4420 */
4421static VALUE
4422ecerr_destination_encoding_name(VALUE self)
4423{
4424 return rb_attr_get(self, id_destination_encoding_name);
4425}
4426
4427/*
4428 * call-seq:
4429 * ecerr.destination_encoding -> string
4430 *
4431 * Returns the destination encoding as an encoding object.
4432 */
4433static VALUE
4434ecerr_destination_encoding(VALUE self)
4435{
4436 return rb_attr_get(self, id_destination_encoding);
4437}
4438
4439/*
4440 * call-seq:
4441 * ecerr.error_char -> string
4442 *
4443 * Returns the one-character string which cause Encoding::UndefinedConversionError.
4444 *
4445 * ec = Encoding::Converter.new("ISO-8859-1", "EUC-JP")
4446 * begin
4447 * ec.convert("\xa0")
4448 * rescue Encoding::UndefinedConversionError
4449 * puts $!.error_char.dump #=> "\xC2\xA0"
4450 * p $!.error_char.encoding #=> #<Encoding:UTF-8>
4451 * end
4452 *
4453 */
4454static VALUE
4455ecerr_error_char(VALUE self)
4456{
4457 return rb_attr_get(self, id_error_char);
4458}
4459
4460/*
4461 * call-seq:
4462 * ecerr.error_bytes -> string
4463 *
4464 * Returns the discarded bytes when Encoding::InvalidByteSequenceError occurs.
4465 *
4466 * ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1")
4467 * begin
4468 * ec.convert("abc\xA1\xFFdef")
4469 * rescue Encoding::InvalidByteSequenceError
4470 * p $! #=> #<Encoding::InvalidByteSequenceError: "\xA1" followed by "\xFF" on EUC-JP>
4471 * puts $!.error_bytes.dump #=> "\xA1"
4472 * puts $!.readagain_bytes.dump #=> "\xFF"
4473 * end
4474 */
4475static VALUE
4476ecerr_error_bytes(VALUE self)
4477{
4478 return rb_attr_get(self, id_error_bytes);
4479}
4480
4481/*
4482 * call-seq:
4483 * ecerr.readagain_bytes -> string
4484 *
4485 * Returns the bytes to be read again when Encoding::InvalidByteSequenceError occurs.
4486 */
4487static VALUE
4488ecerr_readagain_bytes(VALUE self)
4489{
4490 return rb_attr_get(self, id_readagain_bytes);
4491}
4492
4493/*
4494 * call-seq:
4495 * ecerr.incomplete_input? -> true or false
4496 *
4497 * Returns true if the invalid byte sequence error is caused by
4498 * premature end of string.
4499 *
4500 * ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1")
4501 *
4502 * begin
4503 * ec.convert("abc\xA1z")
4504 * rescue Encoding::InvalidByteSequenceError
4505 * p $! #=> #<Encoding::InvalidByteSequenceError: "\xA1" followed by "z" on EUC-JP>
4506 * p $!.incomplete_input? #=> false
4507 * end
4508 *
4509 * begin
4510 * ec.convert("abc\xA1")
4511 * ec.finish
4512 * rescue Encoding::InvalidByteSequenceError
4513 * p $! #=> #<Encoding::InvalidByteSequenceError: incomplete "\xA1" on EUC-JP>
4514 * p $!.incomplete_input? #=> true
4515 * end
4516 */
4517static VALUE
4518ecerr_incomplete_input(VALUE self)
4519{
4520 return rb_attr_get(self, id_incomplete_input);
4521}
4522
4523/*
4524 * Document-class: Encoding::UndefinedConversionError
4525 *
4526 * Raised by Encoding and String methods when a transcoding operation
4527 * fails.
4528 */
4529
4530/*
4531 * Document-class: Encoding::InvalidByteSequenceError
4532 *
4533 * Raised by Encoding and String methods when the string being
4534 * transcoded contains a byte invalid for the either the source or
4535 * target encoding.
4536 */
4537
4538/*
4539 * Document-class: Encoding::ConverterNotFoundError
4540 *
4541 * Raised by transcoding methods when a named encoding does not
4542 * correspond with a known converter.
4543 */
4544
4545void
4546Init_transcode(void)
4547{
4548 transcoder_table = st_init_strcasetable();
4549
4550 id_destination_encoding = rb_intern_const("destination_encoding");
4551 id_destination_encoding_name = rb_intern_const("destination_encoding_name");
4552 id_error_bytes = rb_intern_const("error_bytes");
4553 id_error_char = rb_intern_const("error_char");
4554 id_incomplete_input = rb_intern_const("incomplete_input");
4555 id_readagain_bytes = rb_intern_const("readagain_bytes");
4556 id_source_encoding = rb_intern_const("source_encoding");
4557 id_source_encoding_name = rb_intern_const("source_encoding_name");
4558
4559 sym_invalid = ID2SYM(rb_intern_const("invalid"));
4560 sym_undef = ID2SYM(rb_intern_const("undef"));
4561 sym_replace = ID2SYM(rb_intern_const("replace"));
4562 sym_fallback = ID2SYM(rb_intern_const("fallback"));
4563 sym_xml = ID2SYM(rb_intern_const("xml"));
4564 sym_text = ID2SYM(rb_intern_const("text"));
4565 sym_attr = ID2SYM(rb_intern_const("attr"));
4566
4567 sym_invalid_byte_sequence = ID2SYM(rb_intern_const("invalid_byte_sequence"));
4568 sym_undefined_conversion = ID2SYM(rb_intern_const("undefined_conversion"));
4569 sym_destination_buffer_full = ID2SYM(rb_intern_const("destination_buffer_full"));
4570 sym_source_buffer_empty = ID2SYM(rb_intern_const("source_buffer_empty"));
4571 sym_finished = ID2SYM(rb_intern_const("finished"));
4572 sym_after_output = ID2SYM(rb_intern_const("after_output"));
4573 sym_incomplete_input = ID2SYM(rb_intern_const("incomplete_input"));
4574 sym_universal_newline = ID2SYM(rb_intern_const("universal_newline"));
4575 sym_crlf_newline = ID2SYM(rb_intern_const("crlf_newline"));
4576 sym_cr_newline = ID2SYM(rb_intern_const("cr_newline"));
4577 sym_lf_newline = ID2SYM(rb_intern("lf_newline"));
4578 sym_partial_input = ID2SYM(rb_intern_const("partial_input"));
4579
4580#ifdef ENABLE_ECONV_NEWLINE_OPTION
4581 sym_newline = ID2SYM(rb_intern_const("newline"));
4582 sym_universal = ID2SYM(rb_intern_const("universal"));
4583 sym_crlf = ID2SYM(rb_intern_const("crlf"));
4584 sym_cr = ID2SYM(rb_intern_const("cr"));
4585 sym_lf = ID2SYM(rb_intern_const("lf"));
4586#endif
4587
4588 InitVM(transcode);
4589}
4590
4591void
4592InitVM_transcode(void)
4593{
4594 rb_eUndefinedConversionError = rb_define_class_under(rb_cEncoding, "UndefinedConversionError", rb_eEncodingError);
4595 rb_eInvalidByteSequenceError = rb_define_class_under(rb_cEncoding, "InvalidByteSequenceError", rb_eEncodingError);
4596 rb_eConverterNotFoundError = rb_define_class_under(rb_cEncoding, "ConverterNotFoundError", rb_eEncodingError);
4597
4598 rb_define_method(rb_cString, "encode", str_encode, -1);
4599 rb_define_method(rb_cString, "encode!", str_encode_bang, -1);
4600
4601 rb_cEncodingConverter = rb_define_class_under(rb_cEncoding, "Converter", rb_cObject);
4602 rb_define_alloc_func(rb_cEncodingConverter, econv_s_allocate);
4603 rb_define_singleton_method(rb_cEncodingConverter, "asciicompat_encoding", econv_s_asciicompat_encoding, 1);
4604 rb_define_singleton_method(rb_cEncodingConverter, "search_convpath", econv_s_search_convpath, -1);
4605 rb_define_method(rb_cEncodingConverter, "initialize", econv_init, -1);
4606 rb_define_method(rb_cEncodingConverter, "inspect", econv_inspect, 0);
4607 rb_define_method(rb_cEncodingConverter, "convpath", econv_convpath, 0);
4608 rb_define_method(rb_cEncodingConverter, "source_encoding", econv_source_encoding, 0);
4609 rb_define_method(rb_cEncodingConverter, "destination_encoding", econv_destination_encoding, 0);
4610 rb_define_method(rb_cEncodingConverter, "primitive_convert", econv_primitive_convert, -1);
4611 rb_define_method(rb_cEncodingConverter, "convert", econv_convert, 1);
4612 rb_define_method(rb_cEncodingConverter, "finish", econv_finish, 0);
4613 rb_define_method(rb_cEncodingConverter, "primitive_errinfo", econv_primitive_errinfo, 0);
4614 rb_define_method(rb_cEncodingConverter, "insert_output", econv_insert_output, 1);
4615 rb_define_method(rb_cEncodingConverter, "putback", econv_putback, -1);
4616 rb_define_method(rb_cEncodingConverter, "last_error", econv_last_error, 0);
4617 rb_define_method(rb_cEncodingConverter, "replacement", econv_get_replacement, 0);
4618 rb_define_method(rb_cEncodingConverter, "replacement=", econv_set_replacement, 1);
4619 rb_define_method(rb_cEncodingConverter, "==", econv_equal, 1);
4620
4621 /*
4622 *Mask for invalid byte sequences
4623 */
4624 rb_define_const(rb_cEncodingConverter, "INVALID_MASK", INT2FIX(ECONV_INVALID_MASK));
4625
4626 /*
4627 * Replace invalid byte sequences
4628 */
4629 rb_define_const(rb_cEncodingConverter, "INVALID_REPLACE", INT2FIX(ECONV_INVALID_REPLACE));
4630
4631 /*
4632 * Mask for a valid character in the source encoding but no related
4633 * character(s) in destination encoding.
4634 */
4635 rb_define_const(rb_cEncodingConverter, "UNDEF_MASK", INT2FIX(ECONV_UNDEF_MASK));
4636
4637 /*
4638 * Replace byte sequences that are undefined in the destination encoding.
4639 */
4640 rb_define_const(rb_cEncodingConverter, "UNDEF_REPLACE", INT2FIX(ECONV_UNDEF_REPLACE));
4641
4642 /*
4643 * Replace byte sequences that are undefined in the destination encoding
4644 * with an XML hexadecimal character reference. This is valid for XML
4645 * conversion.
4646 */
4647 rb_define_const(rb_cEncodingConverter, "UNDEF_HEX_CHARREF", INT2FIX(ECONV_UNDEF_HEX_CHARREF));
4648
4649 /*
4650 * Indicates the source may be part of a larger string. See
4651 * primitive_convert for an example.
4652 */
4653 rb_define_const(rb_cEncodingConverter, "PARTIAL_INPUT", INT2FIX(ECONV_PARTIAL_INPUT));
4654
4655 /*
4656 * Stop converting after some output is complete but before all of the
4657 * input was consumed. See primitive_convert for an example.
4658 */
4659 rb_define_const(rb_cEncodingConverter, "AFTER_OUTPUT", INT2FIX(ECONV_AFTER_OUTPUT));
4660
4661 /*
4662 * Decorator for converting CRLF and CR to LF
4663 */
4664 rb_define_const(rb_cEncodingConverter, "UNIVERSAL_NEWLINE_DECORATOR", INT2FIX(ECONV_UNIVERSAL_NEWLINE_DECORATOR));
4665
4666 /*
4667 * Decorator for converting CRLF and CR to LF when writing
4668 */
4669 rb_define_const(rb_cEncodingConverter, "LF_NEWLINE_DECORATOR", INT2FIX(ECONV_LF_NEWLINE_DECORATOR));
4670
4671 /*
4672 * Decorator for converting LF to CRLF
4673 */
4674 rb_define_const(rb_cEncodingConverter, "CRLF_NEWLINE_DECORATOR", INT2FIX(ECONV_CRLF_NEWLINE_DECORATOR));
4675
4676 /*
4677 * Decorator for converting LF to CR
4678 */
4679 rb_define_const(rb_cEncodingConverter, "CR_NEWLINE_DECORATOR", INT2FIX(ECONV_CR_NEWLINE_DECORATOR));
4680
4681 /*
4682 * Escape as XML CharData
4683 */
4684 rb_define_const(rb_cEncodingConverter, "XML_TEXT_DECORATOR", INT2FIX(ECONV_XML_TEXT_DECORATOR));
4685
4686 /*
4687 * Escape as XML AttValue
4688 */
4689 rb_define_const(rb_cEncodingConverter, "XML_ATTR_CONTENT_DECORATOR", INT2FIX(ECONV_XML_ATTR_CONTENT_DECORATOR));
4690
4691 /*
4692 * Escape as XML AttValue
4693 */
4694 rb_define_const(rb_cEncodingConverter, "XML_ATTR_QUOTE_DECORATOR", INT2FIX(ECONV_XML_ATTR_QUOTE_DECORATOR));
4695
4696 rb_define_method(rb_eUndefinedConversionError, "source_encoding_name", ecerr_source_encoding_name, 0);
4697 rb_define_method(rb_eUndefinedConversionError, "destination_encoding_name", ecerr_destination_encoding_name, 0);
4698 rb_define_method(rb_eUndefinedConversionError, "source_encoding", ecerr_source_encoding, 0);
4699 rb_define_method(rb_eUndefinedConversionError, "destination_encoding", ecerr_destination_encoding, 0);
4700 rb_define_method(rb_eUndefinedConversionError, "error_char", ecerr_error_char, 0);
4701
4702 rb_define_method(rb_eInvalidByteSequenceError, "source_encoding_name", ecerr_source_encoding_name, 0);
4703 rb_define_method(rb_eInvalidByteSequenceError, "destination_encoding_name", ecerr_destination_encoding_name, 0);
4704 rb_define_method(rb_eInvalidByteSequenceError, "source_encoding", ecerr_source_encoding, 0);
4705 rb_define_method(rb_eInvalidByteSequenceError, "destination_encoding", ecerr_destination_encoding, 0);
4706 rb_define_method(rb_eInvalidByteSequenceError, "error_bytes", ecerr_error_bytes, 0);
4707 rb_define_method(rb_eInvalidByteSequenceError, "readagain_bytes", ecerr_readagain_bytes, 0);
4708 rb_define_method(rb_eInvalidByteSequenceError, "incomplete_input?", ecerr_incomplete_input, 0);
4709
4710 Init_newline();
4711}
ruby_coderange_type
What rb_enc_str_coderange() returns.
Definition coderange.h:33
#define rb_define_method(klass, mid, func, arity)
Defines klass#mid.
#define rb_define_singleton_method(klass, mid, func, arity)
Defines klass.mid.
VALUE rb_define_class_under(VALUE outer, const char *name, VALUE super)
Defines a class under the namespace of outer.
Definition class.c:1519
int rb_scan_args(int argc, const VALUE *argv, const char *fmt,...)
Retrieves argument from argc and argv to given VALUE references according to the format string.
Definition class.c:3158
#define ECONV_XML_ATTR_QUOTE_DECORATOR
Old name of RUBY_ECONV_XML_ATTR_QUOTE_DECORATOR.
Definition transcode.h:539
#define ECONV_AFTER_OUTPUT
Old name of RUBY_ECONV_AFTER_OUTPUT.
Definition transcode.h:555
#define rb_str_new2
Old name of rb_str_new_cstr.
Definition string.h:1676
#define ENC_CODERANGE_7BIT
Old name of RUBY_ENC_CODERANGE_7BIT.
Definition coderange.h:180
#define ENC_CODERANGE_VALID
Old name of RUBY_ENC_CODERANGE_VALID.
Definition coderange.h:181
#define ECONV_UNIVERSAL_NEWLINE_DECORATOR
Old name of RUBY_ECONV_UNIVERSAL_NEWLINE_DECORATOR.
Definition transcode.h:532
#define REALLOC_N
Old name of RB_REALLOC_N.
Definition memory.h:403
#define ALLOC
Old name of RB_ALLOC.
Definition memory.h:400
#define xfree
Old name of ruby_xfree.
Definition xmalloc.h:58
#define INT2FIX
Old name of RB_INT2FIX.
Definition long.h:48
#define OBJ_FROZEN
Old name of RB_OBJ_FROZEN.
Definition fl_type.h:136
#define ECONV_XML_ATTR_CONTENT_DECORATOR
Old name of RUBY_ECONV_XML_ATTR_CONTENT_DECORATOR.
Definition transcode.h:537
#define rb_str_cat2
Old name of rb_str_cat_cstr.
Definition string.h:1684
#define ECONV_INVALID_MASK
Old name of RUBY_ECONV_INVALID_MASK.
Definition transcode.h:523
#define ECONV_CRLF_NEWLINE_DECORATOR
Old name of RUBY_ECONV_CRLF_NEWLINE_DECORATOR.
Definition transcode.h:533
#define xrealloc
Old name of ruby_xrealloc.
Definition xmalloc.h:56
#define ID2SYM
Old name of RB_ID2SYM.
Definition symbol.h:44
#define OBJ_FREEZE
Old name of RB_OBJ_FREEZE.
Definition fl_type.h:134
#define ECONV_UNDEF_REPLACE
Old name of RUBY_ECONV_UNDEF_REPLACE.
Definition transcode.h:526
#define ECONV_XML_TEXT_DECORATOR
Old name of RUBY_ECONV_XML_TEXT_DECORATOR.
Definition transcode.h:536
#define rb_ary_new4
Old name of rb_ary_new_from_values.
Definition array.h:659
#define ENC_CODERANGE_UNKNOWN
Old name of RUBY_ENC_CODERANGE_UNKNOWN.
Definition coderange.h:179
#define ECONV_CR_NEWLINE_DECORATOR
Old name of RUBY_ECONV_CR_NEWLINE_DECORATOR.
Definition transcode.h:534
#define xmalloc
Old name of ruby_xmalloc.
Definition xmalloc.h:53
#define ECONV_INVALID_REPLACE
Old name of RUBY_ECONV_INVALID_REPLACE.
Definition transcode.h:524
#define T_HASH
Old name of RUBY_T_HASH.
Definition value_type.h:65
#define ALLOC_N
Old name of RB_ALLOC_N.
Definition memory.h:399
#define MBCLEN_CHARFOUND_LEN(ret)
Old name of ONIGENC_MBCLEN_CHARFOUND_LEN.
Definition encoding.h:517
#define rb_exc_new3
Old name of rb_exc_new_str.
Definition error.h:38
#define ECONV_UNDEF_MASK
Old name of RUBY_ECONV_UNDEF_MASK.
Definition transcode.h:525
#define Qtrue
Old name of RUBY_Qtrue.
#define ECONV_PARTIAL_INPUT
Old name of RUBY_ECONV_PARTIAL_INPUT.
Definition transcode.h:554
#define NUM2INT
Old name of RB_NUM2INT.
Definition int.h:44
#define ECONV_ERROR_HANDLER_MASK
Old name of RUBY_ECONV_ERROR_HANDLER_MASK.
Definition transcode.h:522
#define INT2NUM
Old name of RB_INT2NUM.
Definition int.h:43
#define Qnil
Old name of RUBY_Qnil.
#define Qfalse
Old name of RUBY_Qfalse.
#define ENC_CODERANGE_BROKEN
Old name of RUBY_ENC_CODERANGE_BROKEN.
Definition coderange.h:182
#define ECONV_LF_NEWLINE_DECORATOR
Old name of RUBY_ECONV_LF_NEWLINE_DECORATOR.
Definition transcode.h:535
#define T_ARRAY
Old name of RUBY_T_ARRAY.
Definition value_type.h:56
#define NIL_P
Old name of RB_NIL_P.
#define MBCLEN_CHARFOUND_P(ret)
Old name of ONIGENC_MBCLEN_CHARFOUND_P.
Definition encoding.h:516
#define ECONV_UNDEF_HEX_CHARREF
Old name of RUBY_ECONV_UNDEF_HEX_CHARREF.
Definition transcode.h:527
#define NUM2LONG
Old name of RB_NUM2LONG.
Definition long.h:51
#define ECONV_NEWLINE_DECORATOR_MASK
Old name of RUBY_ECONV_NEWLINE_DECORATOR_MASK.
Definition transcode.h:529
#define rb_ary_new2
Old name of rb_ary_new_capa.
Definition array.h:657
#define ENC_CODERANGE_SET(obj, cr)
Old name of RB_ENC_CODERANGE_SET.
Definition coderange.h:186
#define SYMBOL_P
Old name of RB_SYMBOL_P.
Definition value_type.h:88
void rb_exc_raise(VALUE mesg)
Raises an exception in the current thread.
Definition eval.c:664
int rb_typeddata_is_kind_of(VALUE obj, const rb_data_type_t *data_type)
Checks if the given object is of given kind.
Definition error.c:1381
VALUE rb_eTypeError
TypeError exception.
Definition error.c:1431
VALUE rb_eRuntimeError
RuntimeError exception.
Definition error.c:1429
void * rb_check_typeddata(VALUE obj, const rb_data_type_t *data_type)
Identical to rb_typeddata_is_kind_of(), except it raises exceptions instead of returning false.
Definition error.c:1398
VALUE rb_exc_new_str(VALUE etype, VALUE str)
Identical to rb_exc_new_cstr(), except it takes a Ruby's string instead of C's.
Definition error.c:1482
VALUE rb_eEncodingError
EncodingError exception.
Definition error.c:1437
void rb_warning(const char *fmt,...)
Issues a warning.
Definition error.c:497
VALUE rb_obj_class(VALUE obj)
Queries the class of an object.
Definition object.c:264
VALUE rb_cEncoding
Encoding class.
Definition encoding.c:60
VALUE rb_cString
String class.
Definition string.c:84
VALUE rb_to_int(VALUE val)
Identical to rb_check_to_int(), except it raises in case of conversion mismatch.
Definition object.c:3306
Encoding relates APIs.
VALUE rb_str_conv_enc(VALUE str, rb_encoding *from, rb_encoding *to)
Encoding conversion main routine.
Definition string.c:1359
int rb_enc_str_coderange(VALUE str)
Scans the passed string to collect its code range.
Definition string.c:964
int rb_econv_prepare_options(VALUE opthash, VALUE *ecopts, int ecflags)
Identical to rb_econv_prepare_opts(), except it additionally takes the initial value of flags.
Definition transcode.c:2678
VALUE rb_econv_open_exc(const char *senc, const char *denc, int ecflags)
Creates a rb_eConverterNotFoundError exception object (but does not raise).
Definition transcode.c:2125
const char * rb_econv_encoding_to_insert_output(rb_econv_t *ec)
Queries an encoding name which best suits for rb_econv_insert_output()'s last parameter.
Definition transcode.c:1544
int rb_econv_prepare_opts(VALUE opthash, VALUE *ecopts)
Splits a keyword arguments hash (that for instance String#encode took) into a set of enum ruby_econv_...
Definition transcode.c:2723
rb_econv_result_t rb_econv_convert(rb_econv_t *ec, const unsigned char **source_buffer_ptr, const unsigned char *source_buffer_end, unsigned char **destination_buffer_ptr, unsigned char *destination_buffer_end, int flags)
Converts a string from an encoding to another.
Definition transcode.c:1487
rb_econv_result_t
return value of rb_econv_convert()
Definition transcode.h:30
@ econv_incomplete_input
The conversion stopped in middle of reading a character, possibly due to a partial read of a socket e...
Definition transcode.h:69
@ econv_finished
The conversion stopped after converting everything.
Definition transcode.h:57
@ econv_undefined_conversion
The conversion stopped when it found a character in the input which cannot be representable in the ou...
Definition transcode.h:41
@ econv_after_output
The conversion stopped after writing something to somewhere, before reading everything.
Definition transcode.h:63
@ econv_source_buffer_empty
The conversion stopped because there is no input.
Definition transcode.h:51
@ econv_destination_buffer_full
The conversion stopped because there is no destination.
Definition transcode.h:46
@ econv_invalid_byte_sequence
The conversion stopped when it found an invalid sequence.
Definition transcode.h:35
int rb_econv_putbackable(rb_econv_t *ec)
Queries if rb_econv_putback() makes sense, i.e.
Definition transcode.c:1782
int rb_econv_has_convpath_p(const char *from_encoding, const char *to_encoding)
Queries if there is more than one way to convert between the passed two encodings.
Definition transcode.c:3297
rb_econv_t * rb_econv_open(const char *source_encoding, const char *destination_encoding, int ecflags)
Creates a new instance of struct rb_econv_t.
Definition transcode.c:1108
VALUE rb_econv_str_append(rb_econv_t *ec, VALUE src, VALUE dst, int flags)
Identical to rb_econv_str_convert(), except it appends the conversion result to the additionally pass...
Definition transcode.c:1949
VALUE rb_econv_substr_append(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, VALUE dst, int flags)
Identical to rb_econv_str_append(), except it appends only a part of the passed string with conversio...
Definition transcode.c:1940
const char * rb_econv_asciicompat_encoding(const char *encname)
Queries the passed encoding's corresponding ASCII compatible encoding.
Definition transcode.c:1826
int rb_econv_insert_output(rb_econv_t *ec, const unsigned char *str, size_t len, const char *str_encoding)
Appends the passed string to the passed converter's output buffer.
Definition transcode.c:1628
VALUE rb_econv_str_convert(rb_econv_t *ec, VALUE src, int flags)
Identical to rb_econv_convert(), except it takes Ruby's string instead of C's pointer.
Definition transcode.c:1961
rb_econv_t * rb_econv_open_opts(const char *source_encoding, const char *destination_encoding, int ecflags, VALUE ecopts)
Identical to rb_econv_open(), except it additionally takes a hash of optional strings.
Definition transcode.c:2729
int rb_econv_decorate_at_last(rb_econv_t *ec, const char *decorator_name)
Identical to rb_econv_decorate_at_first(), except it adds to the opposite direction.
Definition transcode.c:2007
void rb_econv_binmode(rb_econv_t *ec)
This badly named function does not set the destination encoding to binary, but instead just nullifies...
Definition transcode.c:2024
int rb_econv_decorate_at_first(rb_econv_t *ec, const char *decorator_name)
"Decorate"s a converter.
Definition transcode.c:1990
VALUE rb_str_encode(VALUE str, VALUE to, int ecflags, VALUE ecopts)
Converts the contents of the passed string from its encoding to the passed one.
Definition transcode.c:2992
VALUE rb_econv_make_exception(rb_econv_t *ec)
This function makes sense right after rb_econv_convert() returns.
Definition transcode.c:4361
void rb_econv_check_error(rb_econv_t *ec)
This is a rb_econv_make_exception() + rb_exc_raise() combo.
Definition transcode.c:4367
VALUE rb_econv_substr_convert(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, int flags)
Identical to rb_econv_str_convert(), except it converts only a part of the passed string.
Definition transcode.c:1955
void rb_econv_close(rb_econv_t *ec)
Destructs a converter.
Definition transcode.c:1743
VALUE rb_econv_append(rb_econv_t *ec, const char *bytesrc, long bytesize, VALUE dst, int flags)
Converts the passed C's pointer according to the passed converter, then append the conversion result ...
Definition transcode.c:1877
void rb_econv_putback(rb_econv_t *ec, unsigned char *p, int n)
Puts back the bytes.
Definition transcode.c:1793
int rb_econv_set_replacement(rb_econv_t *ec, const unsigned char *str, size_t len, const char *encname)
Assigns the replacement string.
Definition transcode.c:2289
VALUE rb_funcallv_public(VALUE recv, ID mid, int argc, const VALUE *argv)
Identical to rb_funcallv(), except it only takes public methods into account.
Definition vm_eval.c:1170
VALUE rb_check_array_type(VALUE obj)
Try converting an object to its array representation using its to_ary method, if any.
VALUE rb_ary_new(void)
Allocates a new, empty array.
VALUE rb_ary_push(VALUE ary, VALUE elem)
Special case of rb_ary_cat() that it adds only one element.
VALUE rb_ary_entry(VALUE ary, long off)
Queries an element of an array.
VALUE rb_assoc_new(VALUE car, VALUE cdr)
Identical to rb_ary_new_from_values(), except it expects exactly two parameters.
void rb_ary_store(VALUE ary, long key, VALUE val)
Destructively stores the passed value to the passed array's passed index.
VALUE rb_proc_call(VALUE recv, VALUE args)
Evaluates the passed proc with the passed arguments.
Definition proc.c:1156
VALUE rb_obj_is_method(VALUE recv)
Queries if the given object is a method.
Definition proc.c:1830
VALUE rb_method_call(int argc, const VALUE *argv, VALUE recv)
Evaluates the passed method with the passed arguments.
Definition proc.c:2711
VALUE rb_obj_is_proc(VALUE recv)
Queries if the given object is a proc.
Definition proc.c:126
VALUE rb_str_tmp_new(long len)
Allocates a "temporary" string.
Definition string.c:1763
#define rb_str_new(str, len)
Allocates an instance of rb_cString.
Definition string.h:1499
void rb_str_shared_replace(VALUE dst, VALUE src)
Replaces the contents of the former with the latter.
Definition string.c:1805
size_t rb_str_capacity(VALUE str)
Queries the capacity of the given string.
Definition string.c:1018
VALUE rb_str_new_frozen(VALUE str)
Creates a frozen copy of the string, if necessary.
Definition string.c:1535
VALUE rb_str_dup(VALUE str)
Duplicates a string.
Definition string.c:2014
void rb_str_set_len(VALUE str, long len)
Overwrites the length of the string.
Definition string.c:3418
void rb_str_modify_expand(VALUE str, long capa)
Identical to rb_str_modify(), except it additionally expands the capacity of the receiver.
Definition string.c:2775
VALUE rb_str_dump(VALUE str)
"Inverse" of rb_eval_string().
Definition string.c:7392
VALUE rb_str_buf_new(long capa)
Allocates a "string buffer".
Definition string.c:1735
#define rb_str_new_cstr(str)
Identical to rb_str_new, except it assumes the passed pointer is a pointer to a C string.
Definition string.h:1515
VALUE rb_str_drop_bytes(VALUE str, long len)
Shrinks the given string for the given number of bytes.
Definition string.c:5791
VALUE rb_ivar_set(VALUE obj, ID name, VALUE val)
Identical to rb_iv_set(), except it accepts the name as an ID instead of a C string.
Definition variable.c:2030
int rb_respond_to(VALUE obj, ID mid)
Queries if the object responds to the method.
Definition vm_method.c:3480
void rb_define_alloc_func(VALUE klass, rb_alloc_func_t func)
Sets the allocator function of a class.
VALUE rb_sym2str(VALUE symbol)
Obtain a frozen string representation of a symbol (not including the leading colon).
Definition symbol.c:994
char * ptr
Pointer to the underlying memory region, of at least capa bytes.
Definition io.h:2
int off
Offset inside of ptr.
Definition io.h:5
int len
Length of the buffer.
Definition io.h:8
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
Definition memory.h:372
#define ALLOCA_N(type, n)
Definition memory.h:292
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
Definition memory.h:167
#define MEMMOVE(p1, p2, type, n)
Handy macro to call memmove.
Definition memory.h:384
#define RARRAY_LEN
Just another name of rb_array_len.
Definition rarray.h:51
#define RARRAY_AREF(a, i)
Definition rarray.h:403
#define DATA_PTR(obj)
Convenient getter macro.
Definition rdata.h:67
#define StringValue(v)
Ensures that the parameter object is a String.
Definition rstring.h:66
#define StringValueCStr(v)
Identical to StringValuePtr, except it additionally checks for the contents for viability as a C stri...
Definition rstring.h:89
#define TypedData_Get_Struct(obj, type, data_type, sval)
Obtains a C struct from inside of a wrapper Ruby object.
Definition rtypeddata.h:649
#define TypedData_Wrap_Struct(klass, data_type, sval)
Converts sval, a pointer to your struct, into a Ruby object.
Definition rtypeddata.h:461
struct rb_data_type_struct rb_data_type_t
This is the struct that holds necessary info for a struct.
Definition rtypeddata.h:205
const char * rb_obj_classname(VALUE obj)
Queries the name of the class of the passed object.
Definition variable.c:515
#define InitVM(ext)
This macro is for internal use.
Definition ruby.h:231
#define RTEST
This is an old name of RB_TEST.
Definition st.h:79
Definition string.c:8285
Definition transcode.c:177
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40