gnu-fdl-en-cz.sh: employ Pango formatting

Also add collation fields.

dicts/gnu-fdl-en-cz.sh

@@ -2,20 +2,28 @@
 # GNU/FDL English-Czech dictionary, see https://www.svobodneslovniky.cz/
 curl -Lo- https://www.svobodneslovniky.cz/data/en-cs.txt.gz | \
 zcat | grep -v ^# | sed 's/\\//g' | perl -CSD -F\\t -le '
-	sub e { shift =~ s/\\/\\\\/gr =~ s/\n/\\n/gr =~ s/\t/\\t/gr }
+	sub tabesc { shift =~ s/\\/\\\\/gr =~ s/\n/\\n/gr =~ s/\t/\\t/gr }
 	sub w {
-		open(my $f, "|-", "tabfile gnu-fdl-$_[0]") or die $!;
-		print $f e($k) . "\t" . e(join("\n", @$v))
-			while ($k, $v) = each %{$_[1]};
+		my ($name, $dict, $collation) = @_;
+		open(my $f, "|-", "tabfile", "--pango", "--collation=$collation",
+			"--website=https://www.svobodneslovniky.cz",
+			"gnu-fdl-$name") or die $!;
+		print $f tabesc($keyword) . "\t" . tabesc(join("\n", @$defs))
+			while ($keyword, $defs) = each %{$dict};
 		close($f);
 	}
-	my ($en, $cz, $notes, $special, $translator) = @F;
-	if ($cz) {
+	sub xmlesc { shift =~ s/&/&amp;/gr =~ s/</&lt;/gr =~ s/>/&gt;/gr }
+	sub entry {
+		my ($definition, $notes) = map {xmlesc($_)} @_;
+		$notes ? "$definition <i>$notes</i>" : $definition;
+	}
+	my ($en, $cs, $notes, $special, $translator) = @F;
+	if ($cs) {
 		$notes =~ s/\w+:\s?//g;          # remove word classes
 		$notes =~ s/(\w+\.)(?!])/($1)/;  # quote "pl."
-		push(@{$encz{$en}}, $notes ? "$cz " . $notes : $cz);
-		push(@{$czen{$cz}}, $notes ? "$en " . $notes : $en);
+		push(@{$encs{$en}}, entry($cs, $notes));
+		push(@{$csen{$cs}}, entry($en, $notes));
 	} END {
-		w("en-cz", \%encz);
-		w("cz-en", \%czen);
+		w("en-cz", \%encs, "en");
+		w("cz-en", \%csen, "cs");
 	}'

src/sdtui.c

@@ -48,6 +48,10 @@
 #define TOP_BAR_CUTOFF  2               ///< How many lines are reserved on top
 #define APP_TITLE  PROJECT_NAME " "     ///< Left top corner
 
+#ifndef A_ITALIC
+#define A_ITALIC 0
+#endif
+
 // --- Utilities ---------------------------------------------------------------
 
 static size_t
@@ -193,8 +197,8 @@ typedef struct application              Application;
 struct view_entry
 {
 	gchar     * word;                   ///< Word
-	gchar  ** definitions;              ///< Word definition entries
-	gsize     definitions_length;       ///< Length of the @a definitions array
+	GPtrArray * definitions;            ///< Word definition entries (gchar *)
+	GPtrArray * formatting;             ///< chtype * or NULL per definition
 };
 
 struct dictionary
@@ -269,14 +273,75 @@ app_char_width (Application *app, gunichar c)
 
 /// Splits the entry and adds it to a pointer array.
 static void
-view_entry_split_add (GPtrArray *out, const gchar *text)
+view_entry_split_add (ViewEntry *ve, const gchar *text, const chtype *attrs)
 {
 	const gchar *p = text, *nl;
 	for (; (nl = strchr (p, '\n')); p = nl + 1)
 		if (nl != p)
-			g_ptr_array_add (out, g_strndup (p, nl - p));
+		{
+			g_ptr_array_add (ve->definitions, g_strndup (p, nl - p));
+			g_ptr_array_add (ve->formatting, !attrs ? NULL
+				: g_memdup2 (attrs + (p - text), (nl - p) * sizeof *attrs));
+		}
 	if (*p)
-		g_ptr_array_add (out, g_strdup (p));
+	{
+		g_ptr_array_add (ve->definitions, g_strdup (p));
+		g_ptr_array_add (ve->formatting, !attrs ? NULL
+			: g_memdup2 (attrs + (p - text), strlen (p) * sizeof *attrs));
+	}
+}
+
+static chtype
+app_pango_iterator_to_attrs (PangoAttrIterator *iterator)
+{
+	chtype attrs = 0;
+	PangoAttrInt *attr = NULL;
+	if ((attr = (PangoAttrInt *) pango_attr_iterator_get (iterator,
+			PANGO_ATTR_WEIGHT)) && attr->value >= PANGO_WEIGHT_BOLD)
+		attrs |= A_BOLD;
+	if ((attr = (PangoAttrInt *) pango_attr_iterator_get (iterator,
+			PANGO_ATTR_UNDERLINE)) && attr->value == PANGO_UNDERLINE_SINGLE)
+		attrs |= A_UNDERLINE;
+	if ((attr = (PangoAttrInt *) pango_attr_iterator_get (iterator,
+			PANGO_ATTR_STYLE)) && attr->value == PANGO_STYLE_ITALIC)
+		attrs |= A_ITALIC;
+	return attrs;
+}
+
+static void
+view_entry_split_add_pango (ViewEntry *ve, const gchar *markup)
+{
+	// This function skips leading whitespace, but it's the canonical one
+	gchar *text = NULL;
+	PangoAttrList *attrs = NULL;
+	if (!pango_parse_markup (markup, -1, 0, &attrs, &text, NULL, NULL))
+	{
+		gchar *replacement = g_strdup_printf ("<%s>", _("error in entry"));
+		view_entry_split_add (ve, replacement, NULL);
+		g_free (replacement);
+		return;
+	}
+
+	PangoAttrIterator *iterator = pango_attr_list_get_iterator (attrs);
+	chtype *formatting = g_malloc0_n (strlen (text), sizeof *formatting);
+	do
+	{
+		gint start = 0, end = 0;
+		pango_attr_iterator_range (iterator, &start, &end);
+		if (end == G_MAXINT)
+			end = strlen (text);
+
+		chtype attrs = app_pango_iterator_to_attrs (iterator);
+		while (start < end)
+			formatting[start++] = attrs;
+	}
+	while (pango_attr_iterator_next (iterator));
+
+	view_entry_split_add (ve, text, formatting);
+	g_free (formatting);
+	pango_attr_iterator_destroy (iterator);
+	pango_attr_list_unref (attrs);
+	g_free (text);
 }
 
 /// Decomposes a dictionary entry into the format we want.
@@ -291,29 +356,22 @@ view_entry_new (StardictIterator *iterator)
 	StardictEntry *entry = stardict_iterator_get_entry (iterator);
 	g_return_val_if_fail (entry != NULL, NULL);
 
-	GPtrArray *definitions = g_ptr_array_new ();
-	const GList *fields = stardict_entry_get_fields (entry);
+	ve->definitions = g_ptr_array_new_with_free_func (g_free);
+	ve->formatting = g_ptr_array_new_with_free_func (g_free);
 	gboolean found_anything_displayable = FALSE;
-	while (fields)
+	for (const GList *fields = stardict_entry_get_fields (entry); fields; )
 	{
 		const StardictEntryField *field = fields->data;
 		switch (field->type)
 		{
 		case STARDICT_FIELD_MEANING:
-			view_entry_split_add (definitions, field->data);
+			view_entry_split_add (ve, field->data, NULL);
 			found_anything_displayable = TRUE;
 			break;
 		case STARDICT_FIELD_PANGO:
-		{
-			char *text;
-			if (!pango_parse_markup (field->data, -1,
-				0, NULL, &text, NULL, NULL))
-				text = g_strdup_printf ("<%s>", _("error in entry"));
-			view_entry_split_add (definitions, text);
+			view_entry_split_add_pango (ve, field->data);
 			found_anything_displayable = TRUE;
-			g_free (text);
 			break;
-		}
 		case STARDICT_FIELD_PHONETIC:
 			g_string_append_printf (word, " /%s/", (const gchar *) field->data);
 			break;
@@ -326,13 +384,10 @@ view_entry_new (StardictIterator *iterator)
 	g_object_unref (entry);
 
 	if (!found_anything_displayable)
-		g_ptr_array_add (definitions,
+		g_ptr_array_add (ve->definitions,
 			g_strdup_printf ("<%s>", _("no usable field found")));
 
 	ve->word = g_string_free (word, FALSE);
-	ve->definitions_length = definitions->len;
-	g_ptr_array_add (definitions, NULL);
-	ve->definitions = (gchar **) g_ptr_array_free (definitions, FALSE);
 	return ve;
 }
 
@@ -341,7 +396,8 @@ static void
 view_entry_free (ViewEntry *ve)
 {
 	g_free (ve->word);
-	g_strfreev (ve->definitions);
+	g_ptr_array_free (ve->definitions, TRUE);
+	g_ptr_array_free (ve->formatting, TRUE);
 	g_slice_free1 (sizeof *ve, ve);
 }
 
@@ -396,7 +452,7 @@ app_reload_view (Application *self)
 	while (remains > 0 && stardict_iterator_is_valid (iterator))
 	{
 		ViewEntry *entry = view_entry_new (iterator);
-		remains -= entry->definitions_length;
+		remains -= entry->definitions->len;
 		g_ptr_array_add (self->entries, entry);
 		stardict_iterator_next (iterator);
 	}
@@ -429,9 +485,7 @@ app_load_color (Application *self, GKeyFile *kf, const gchar *name, int id)
 		else if (!strcmp (*it, "ul"))      attrs.attrs |= A_UNDERLINE;
 		else if (!strcmp (*it, "blink"))   attrs.attrs |= A_BLINK;
 		else if (!strcmp (*it, "reverse")) attrs.attrs |= A_REVERSE;
-#ifdef A_ITALIC
 		else if (!strcmp (*it, "italic"))  attrs.attrs |= A_ITALIC;
-#endif  // A_ITALIC
 	}
 	g_strfreev (values);
 
@@ -778,10 +832,11 @@ row_buffer_init (RowBuffer *self, Application *app)
 
 /// Replace invalid chars and push all codepoints to the array w/ attributes.
 static void
-row_buffer_append (RowBuffer *self, const gchar *str, chtype attrs)
+row_buffer_append_length (RowBuffer *self,
+	const gchar *text, glong length, chtype attrs)
 {
 	glong ucs4_len;
-	gunichar *ucs4 = g_utf8_to_ucs4_fast (str, -1, &ucs4_len);
+	gunichar *ucs4 = g_utf8_to_ucs4_fast (text, length, &ucs4_len);
 	for (glong i = 0; i < ucs4_len; i++)
 	{
 		// XXX: this is very crude as it disrespects combining marks
@@ -794,6 +849,47 @@ row_buffer_append (RowBuffer *self, const gchar *str, chtype attrs)
 	g_free (ucs4);
 }
 
+static void
+row_buffer_append (RowBuffer *self, const gchar *text, chtype attrs)
+{
+	row_buffer_append_length (self, text, -1, attrs);
+}
+
+/// Combine attributes, taking care to replace colour bits entirely
+static void
+row_buffer_merge_attributes (chtype *target, int merged)
+{
+	if (merged & A_COLOR)
+		*target = (*target & ~A_COLOR) | merged;
+	else
+		*target |= merged;
+}
+
+static void
+row_buffer_append_with_formatting (RowBuffer *self,
+	const gchar *text, const chtype *formatting, chtype default_attrs)
+{
+	if (!formatting)
+	{
+		row_buffer_append (self, text, default_attrs);
+		return;
+	}
+
+	while (*text)
+	{
+		glong chunk_length = 1;
+		while (text[chunk_length] && formatting[chunk_length] == *formatting)
+			chunk_length++;
+
+		chtype merged = default_attrs;
+		row_buffer_merge_attributes (&merged, *formatting);
+		row_buffer_append_length (self, text, chunk_length, merged);
+
+		text += chunk_length;
+		formatting += chunk_length;
+	}
+}
+
 /// Pop as many codepoints as needed to free up "space" character cells.
 /// Given the suffix nature of combining marks, this should work pretty fine.
 static gint
@@ -996,16 +1092,6 @@ app_show_help (Application *self)
 	app_show_message (self, lines, G_N_ELEMENTS (lines));
 }
 
-/// Combine attributes, taking care to replace colour bits entirely
-static void
-app_merge_attributes (int *target, int merged)
-{
-	if (merged & A_COLOR)
-		*target = (*target & ~A_COLOR) | merged;
-	else
-		*target |= merged;
-}
-
 /// Redraw the dictionary view.
 static void
 app_redraw_view (Application *self)
@@ -1028,16 +1114,16 @@ app_redraw_view (Application *self)
 	for (i = 0; i < self->entries->len; i++)
 	{
 		ViewEntry *ve = g_ptr_array_index (self->entries, i);
-		for (; k < ve->definitions_length; k++)
+		for (; k < ve->definitions->len; k++)
 		{
-			int attrs = ((self->top_position + i) & 1)
+			chtype attrs = ((self->top_position + i) & 1)
 				? APP_ATTR (ODD) : APP_ATTR (EVEN);
 
 			if (shown == self->selected)
-				app_merge_attributes (&attrs, self->focused
+				row_buffer_merge_attributes (&attrs, self->focused
 					? APP_ATTR (SELECTION) : APP_ATTR (DEFOCUSED));
 
-			gboolean last = k + 1 == ve->definitions_length;
+			gboolean last = k + 1 == ve->definitions->len;
 			if (last && self->underline_last)
 				attrs |= A_UNDERLINE;
 
@@ -1059,7 +1145,9 @@ app_redraw_view (Application *self)
 
 			row_buffer_init (&buf, self);
 			row_buffer_append (&buf, " ", attrs);
-			row_buffer_append (&buf, ve->definitions[k], attrs);
+			row_buffer_append_with_formatting (&buf,
+				g_ptr_array_index (ve->definitions, k),
+				g_ptr_array_index (ve->formatting, k), attrs);
 			row_buffer_finish (&buf, COLS - left_width, attrs);
 
 			if ((gint) ++shown == LINES - TOP_BAR_CUTOFF)
@@ -1104,7 +1192,7 @@ app_count_view_items (Application *self)
 	for (i = 0; i < self->entries->len; i++)
 	{
 		ViewEntry *entry = g_ptr_array_index (self->entries, i);
-		n_definitions += entry->definitions_length;
+		n_definitions += entry->definitions->len;
 	}
 	return n_definitions;
 }
@@ -1128,16 +1216,16 @@ app_scroll_up (Application *self, guint n)
 			break;
 
 		ViewEntry *ve = prepend_entry (self, --self->top_position);
-		self->top_offset = ve->definitions_length - 1;
-		n_definitions += ve->definitions_length;
+		self->top_offset = ve->definitions->len - 1;
+		n_definitions += ve->definitions->len;
 
 		// Remove the last entry if not shown
 		ViewEntry *last_entry =
 			g_ptr_array_index (self->entries, self->entries->len - 1);
 		if ((gint) (n_definitions - self->top_offset
-			- last_entry->definitions_length) >= LINES - TOP_BAR_CUTOFF)
+			- last_entry->definitions->len) >= LINES - TOP_BAR_CUTOFF)
 		{
-			n_definitions -= last_entry->definitions_length;
+			n_definitions -= last_entry->definitions->len;
 			g_ptr_array_remove_index_fast
 				(self->entries, self->entries->len - 1);
 		}
@@ -1161,9 +1249,9 @@ app_scroll_down (Application *self, guint n)
 		guint to_be_definitions = n_definitions;
 
 		ViewEntry *first_entry = g_ptr_array_index (self->entries, 0);
-		if (++to_be_offset >= first_entry->definitions_length)
+		if (++to_be_offset >= first_entry->definitions->len)
 		{
-			to_be_definitions -= first_entry->definitions_length;
+			to_be_definitions -= first_entry->definitions->len;
 			to_be_offset = 0;
 		}
 		if ((gint) (to_be_definitions - to_be_offset) < LINES - TOP_BAR_CUTOFF)
@@ -1174,7 +1262,7 @@ app_scroll_down (Application *self, guint n)
 				break;
 
 			g_ptr_array_add (self->entries, new_entry);
-			to_be_definitions += new_entry->definitions_length;
+			to_be_definitions += new_entry->definitions->len;
 		}
 		if (to_be_offset == 0)
 		{
@@ -1205,7 +1293,7 @@ app_one_entry_up (Application *self)
 	for (i = 0; i < self->entries->len; i++)
 	{
 		ViewEntry *ve = g_ptr_array_index (self->entries, i);
-		gint new_first = first + ve->definitions_length;
+		gint new_first = first + ve->definitions->len;
 		if (new_first >= (gint) self->selected)
 			break;
 		first = new_first;
@@ -1233,7 +1321,7 @@ app_one_entry_down (Application *self)
 	for (i = 0; i < self->entries->len; i++)
 	{
 		ViewEntry *ve = g_ptr_array_index (self->entries, i);
-		first += ve->definitions_length;
+		first += ve->definitions->len;
 		if (first > (gint) self->selected)
 			break;
 	}
@@ -1369,9 +1457,9 @@ app_get_current_definition (Application *self)
 	for (guint i = 0; i < self->entries->len; i++)
 	{
 		ViewEntry *ve = g_ptr_array_index (self->entries, i);
-		if (offset < ve->definitions_length)
-			return ve->definitions[offset];
-		offset -= ve->definitions_length;
+		if (offset < ve->definitions->len)
+			return g_ptr_array_index (ve->definitions, offset);
+		offset -= ve->definitions->len;
 	}
 	return NULL;
 }