converter xdfx to csv

This commit is contained in:
Морозов Андрей 2022-05-02 20:05:07 +04:00
parent 8cf21b5e62
commit 07cda3e10b
1 changed files with 92 additions and 33 deletions

119
main.go
View File

@ -2,37 +2,47 @@ package main
import (
"bufio"
"encoding/csv"
"fmt"
"log"
"os"
"strings"
"sync"
"time"
)
type Words struct {
word string
translation string
transcription string
}
var (
full_name string
lang_from string
lang_to string
)
const (
FIRST_LINES = 5
)
func main() {
f, err := os.OpenFile("dict.xdxf", os.O_RDONLY, os.ModePerm)
fcsv, err := os.Create("./dict.csv")
if err != nil {
fmt.Println(err)
}
defer fcsv.Close()
writer := csv.NewWriter(fcsv)
writer.Comma = '|'
fxdfx, err := os.OpenFile("dict.xdxf", os.O_RDONLY, os.ModePerm)
if err != nil {
log.Fatalf("open file error: %v", err)
return
}
defer f.Close()
defer fxdfx.Close()
fullFile(f)
fullFile(fxdfx, writer)
}
func fullFile(file *os.File) {
sc := bufio.NewScanner(file)
func fullFile(fxdfx *os.File, fcsv *csv.Writer) {
sc := bufio.NewScanner(fxdfx)
/*for sc.Scan() {
fmt.Println(sc.Text())
}*/
@ -64,34 +74,83 @@ func fullFile(file *os.File) {
fmt.Printf("Dict name: %s.\n", full_name)
fmt.Printf("From %s to %s. \n", lang_from, lang_to)
fmt.Println(first)
//fmt.Println(first)
// read file line by line
var (
word string
translation string
transcription string
)
for sc.Scan() {
line := sc.Text()
if strings.Contains(line, "<ar>") {
word = ""
translation = ""
transcription = ""
fmt.Println(word, translation, transcription)
neww := new(Words)
for {
first = strings.TrimSpace(first)
if first == "" {
if sc.Scan() {
first += sc.Text()
} else {
break
}
}
if strings.Contains(first, "<ar>") {
neww.word = ""
neww.translation = ""
neww.transcription = ""
//fmt.Println(neww)
}
if strings.Contains(first, "<k>") {
_, neww.word, _ = strings.Cut(first, "<k>")
if strings.Contains(neww.word, "</k>") {
neww.word, first, _ = strings.Cut(neww.word, "</k>")
first = strings.TrimSpace(first)
if first == "" {
if sc.Scan() {
first += sc.Text()
} else {
break
}
}
}
} else if strings.Contains(first, "<tr>") {
_, neww.transcription, _ = strings.Cut(first, "<tr>")
if strings.Contains(neww.transcription, "</tr>") {
neww.transcription, first, _ = strings.Cut(neww.transcription, "</tr>")
first = strings.TrimSpace(first)
if first == "" {
if sc.Scan() {
first += sc.Text()
} else {
break
}
}
}
} else if strings.Contains(first, "</ar>") {
neww.translation, first, _ = strings.Cut(first, "</ar>")
first = strings.TrimSpace(first)
first = strings.TrimSpace(first)
if first == "" {
if sc.Scan() {
//fmt.Printf("Word: %s. Transcription: %s. Translation: %s.\n", neww.word, neww.transcription, neww.translation)
err := fcsv.Write([]string{neww.word, neww.transcription, neww.translation})
if err != nil {
fmt.Println(err)
}
first += sc.Text()
} else {
break
}
}
} else {
if sc.Scan() {
//fmt.Printf("Word: %s. Transcription: %s. Translation: %s.\n", neww.word, neww.transcription, neww.translation)
first += sc.Text()
} else {
break
}
if strings.Contains(line, "<k>") {
_, word, _ = strings.Cut(line, "<k>")
}
fmt.Println(sc.Text())
}
}
func processChunk(chunk []byte, linesPool *sync.Pool, stringPool *sync.Pool, start time.Time, end time.Time) {
/*func processChunk(chunk []byte, linesPool *sync.Pool, stringPool *sync.Pool, start time.Time, end time.Time) {
}
//readFile opens file in chunks and process to other func. https://medium.com/swlh/processing-16gb-file-in-seconds-go-lang-3982c235dfa2
func readFile() {
}
}*/