diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3995c4f..f0f1389 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,7 +6,7 @@ on: - cron: "0 0 * * 0" push: branches: - - master + - main paths-ignore: - "README.md" - ".gitignore" @@ -16,18 +16,16 @@ on: jobs: build_geoip: - name: Build geoip.dat (from refilter-geoip) + name: Build geoip.dat runs-on: ubuntu-latest steps: - - name: Checkout refilter-geoip + - name: Checkout repository uses: actions/checkout@v4 - with: - path: ./refilter-geoip - name: Set up Go uses: actions/setup-go@v5 with: - go-version-file: ./refilter-geoip/go.mod + go-version: "1.19" - name: Set variables for geoip run: | @@ -35,12 +33,14 @@ jobs: echo "RELEASE_NAME=$(date +%Y%m%d%H%M)" >> $GITHUB_ENV shell: bash - - name: Build geoip.dat - run: go run ./refilter-geoip/main.go + - name: Build and run geoip.dat + run: | + go build -o geoip_gen main.go + ./geoip_gen -m geoip - name: Generate sha256 checksum for geoip.dat files run: | - cd ./refilter-geoip/output/dat || exit 1 + cd ./output/dat || exit 1 for name in $(ls *.dat); do sha256sum ${name} > ./${name}.sha256sum done @@ -48,7 +48,7 @@ jobs: - name: Move geoip files to publish directory run: | mkdir -p publish - mv ./refilter-geoip/output/dat/*.dat ./refilter-geoip/output/dat/*.sha256sum ./publish + mv ./output/dat/*.dat ./output/dat/*.sha256sum ./publish - name: Release and upload geoip assets run: | @@ -59,18 +59,16 @@ jobs: GITHUB_TOKEN: ${{ github.token }} build_geosite: - name: Build geosite.dat (from refilter-domain) + name: Build geosite.dat runs-on: ubuntu-latest steps: - - name: Checkout refilter-domain + - name: Checkout repository uses: actions/checkout@v4 - with: - path: ./refilter-domain - - name: Setup Go + - name: Set up Go uses: actions/setup-go@v5 with: - go-version-file: ./refilter-domain/go.mod + go-version: "1.19" - name: Set variables for geosite run: | @@ -81,8 +79,10 @@ jobs: - name: Use local domains file run: mkdir -p data && cp ./domains_all.lst ./data/refilter - - name: Build geosite.dat - run: go run ./refilter-domain/main.go --exportlists=refilter --outputname=geosite.dat + - name: Build and run geosite.dat + run: | + go build -o geosite_gen main.go + ./geosite_gen -m geosite --exportlists=refilter --outputname=geosite.dat - name: Generate sha256 hash for geosite.dat run: sha256sum geosite.dat > geosite.dat.sha256sum diff --git a/config.json b/config.json new file mode 100644 index 0000000..0d69df9 --- /dev/null +++ b/config.json @@ -0,0 +1,27 @@ +{ + "input": [ + { + "type": "text", + "action": "add", + "args": { + "name": "refilter", + "uri": "https://raw.githubusercontent.com/1andrevich/Re-filter-lists/refs/heads/main/ipsum.lst", + "onlyIPType": "ipv4" + } + }, + { + "type": "private", + "action": "add" + } + ], + "output": [ + { + "type": "v2rayGeoIPDat", + "action": "output", + "args": { + "outputName": "geoip.dat", + "wantedList": ["refilter", "private"] + } + } + ] +} diff --git a/lib/config.go b/lib/config.go new file mode 100644 index 0000000..20e5137 --- /dev/null +++ b/lib/config.go @@ -0,0 +1,128 @@ +package lib + +import ( + "encoding/json" + "errors" + "fmt" + "strings" +) + +var ( + inputConfigCreatorCache = make(map[string]inputConfigCreator) + outputConfigCreatorCache = make(map[string]outputConfigCreator) +) + +type inputConfigCreator func(Action, json.RawMessage) (InputConverter, error) + +type outputConfigCreator func(Action, json.RawMessage) (OutputConverter, error) + +func RegisterInputConfigCreator(id string, fn inputConfigCreator) error { + id = strings.ToLower(id) + if _, found := inputConfigCreatorCache[id]; found { + return errors.New("config creator has already been registered") + } + inputConfigCreatorCache[id] = fn + return nil +} + +func createInputConfig(id string, action Action, data json.RawMessage) (InputConverter, error) { + id = strings.ToLower(id) + fn, found := inputConfigCreatorCache[id] + if !found { + return nil, errors.New("unknown config type") + } + return fn(action, data) +} + +func RegisterOutputConfigCreator(id string, fn outputConfigCreator) error { + id = strings.ToLower(id) + if _, found := outputConfigCreatorCache[id]; found { + return errors.New("config creator has already been registered") + } + outputConfigCreatorCache[id] = fn + return nil +} + +func createOutputConfig(id string, action Action, data json.RawMessage) (OutputConverter, error) { + id = strings.ToLower(id) + fn, found := outputConfigCreatorCache[id] + if !found { + return nil, errors.New("unknown config type") + } + return fn(action, data) +} + +type config struct { + Input []*inputConvConfig `json:"input"` + Output []*outputConvConfig `json:"output"` +} + +type inputConvConfig struct { + iType string + action Action + converter InputConverter +} + +func (i *inputConvConfig) UnmarshalJSON(data []byte) error { + var temp struct { + Type string `json:"type"` + Action Action `json:"action"` + Args json.RawMessage `json:"args"` + } + + if err := json.Unmarshal(data, &temp); err != nil { + return err + } + + if !ActionsRegistry[temp.Action] { + return fmt.Errorf("invalid action %s in type %s", temp.Action, temp.Type) + } + + config, err := createInputConfig(temp.Type, temp.Action, temp.Args) + if err != nil { + return err + } + + i.iType = config.GetType() + i.action = config.GetAction() + i.converter = config + + return nil +} + +type outputConvConfig struct { + iType string + action Action + converter OutputConverter +} + +func (i *outputConvConfig) UnmarshalJSON(data []byte) error { + var temp struct { + Type string `json:"type"` + Action Action `json:"action"` + Args json.RawMessage `json:"args"` + } + + if err := json.Unmarshal(data, &temp); err != nil { + return err + } + + if temp.Action == "" { + temp.Action = ActionOutput + } + + if !ActionsRegistry[temp.Action] { + return fmt.Errorf("invalid action %s in type %s", temp.Action, temp.Type) + } + + config, err := createOutputConfig(temp.Type, temp.Action, temp.Args) + if err != nil { + return err + } + + i.iType = config.GetType() + i.action = config.GetAction() + i.converter = config + + return nil +} diff --git a/lib/error.go b/lib/error.go new file mode 100644 index 0000000..5778e96 --- /dev/null +++ b/lib/error.go @@ -0,0 +1,13 @@ +package lib + +import "errors" + +var ( + ErrDuplicatedConverter = errors.New("duplicated converter") + ErrUnknownAction = errors.New("unknown action") + ErrInvalidIPType = errors.New("invalid IP type") + ErrInvalidIP = errors.New("invalid IP address") + ErrInvalidIPLength = errors.New("invalid IP address length") + ErrInvalidIPNet = errors.New("invalid IPNet address") + ErrInvalidPrefixType = errors.New("invalid prefix type") +) diff --git a/lib/func.go b/lib/func.go new file mode 100644 index 0000000..8cef3ce --- /dev/null +++ b/lib/func.go @@ -0,0 +1,43 @@ +package lib + +import ( + "fmt" + "strings" +) + +var ( + inputConverterMap = make(map[string]InputConverter) + outputConverterMap = make(map[string]OutputConverter) +) + +func ListInputConverter() { + fmt.Println("All available input formats:") + for name, ic := range inputConverterMap { + fmt.Printf(" - %s (%s)\n", name, ic.GetDescription()) + } +} + +func RegisterInputConverter(name string, c InputConverter) error { + name = strings.TrimSpace(name) + if _, ok := inputConverterMap[name]; ok { + return ErrDuplicatedConverter + } + inputConverterMap[name] = c + return nil +} + +func ListOutputConverter() { + fmt.Println("All available output formats:") + for name, oc := range outputConverterMap { + fmt.Printf(" - %s (%s)\n", name, oc.GetDescription()) + } +} + +func RegisterOutputConverter(name string, c OutputConverter) error { + name = strings.TrimSpace(name) + if _, ok := outputConverterMap[name]; ok { + return ErrDuplicatedConverter + } + outputConverterMap[name] = c + return nil +} diff --git a/lib/instance.go b/lib/instance.go new file mode 100644 index 0000000..5631766 --- /dev/null +++ b/lib/instance.go @@ -0,0 +1,65 @@ +package lib + +import ( + "encoding/json" + "errors" + "os" +) + +type Instance struct { + config *config + input []InputConverter + output []OutputConverter +} + +func NewInstance() (*Instance, error) { + return &Instance{ + config: new(config), + input: make([]InputConverter, 0), + output: make([]OutputConverter, 0), + }, nil +} + +func (i *Instance) Init(configFile string) error { + content, err := os.ReadFile(configFile) + if err != nil { + return err + } + + if err := json.Unmarshal(content, &i.config); err != nil { + return err + } + + for _, input := range i.config.Input { + i.input = append(i.input, input.converter) + } + + for _, output := range i.config.Output { + i.output = append(i.output, output.converter) + } + + return nil +} + +func (i *Instance) Run() error { + if len(i.input) == 0 || len(i.output) == 0 { + return errors.New("input type and output type must be specified") + } + + var err error + container := NewContainer() + for _, ic := range i.input { + container, err = ic.Input(container) + if err != nil { + return err + } + } + + for _, oc := range i.output { + if err := oc.Output(container); err != nil { + return err + } + } + + return nil +} diff --git a/lib/lib.go b/lib/lib.go new file mode 100644 index 0000000..d1d23ab --- /dev/null +++ b/lib/lib.go @@ -0,0 +1,458 @@ +package lib + +import ( + "fmt" + "log" + "net" + "net/netip" + "strings" + "sync" + + "go4.org/netipx" +) + +const ( + ActionAdd Action = "add" + ActionRemove Action = "remove" + ActionOutput Action = "output" + + IPv4 IPType = "ipv4" + IPv6 IPType = "ipv6" +) + +var ActionsRegistry = map[Action]bool{ + ActionAdd: true, + ActionRemove: true, + ActionOutput: true, +} + +type Action string + +type IPType string + +type Typer interface { + GetType() string +} + +type Actioner interface { + GetAction() Action +} + +type Descriptioner interface { + GetDescription() string +} + +type InputConverter interface { + Typer + Actioner + Descriptioner + Input(Container) (Container, error) +} + +type OutputConverter interface { + Typer + Actioner + Descriptioner + Output(Container) error +} + +type Entry struct { + name string + mu *sync.Mutex + ipv4Builder *netipx.IPSetBuilder + ipv6Builder *netipx.IPSetBuilder +} + +func NewEntry(name string) *Entry { + return &Entry{ + name: strings.ToUpper(strings.TrimSpace(name)), + mu: new(sync.Mutex), + ipv4Builder: new(netipx.IPSetBuilder), + ipv6Builder: new(netipx.IPSetBuilder), + } +} + +func (e *Entry) GetName() string { + return e.name +} + +func (e *Entry) hasIPv4Builder() bool { + return e.ipv4Builder != nil +} + +func (e *Entry) hasIPv6Builder() bool { + return e.ipv6Builder != nil +} + +func (e *Entry) processPrefix(src any) (*netip.Prefix, IPType, error) { + switch src := src.(type) { + case net.IP: + ip, ok := netipx.FromStdIP(src) + if !ok { + return nil, "", ErrInvalidIP + } + switch { + case ip.Is4(): + prefix := netip.PrefixFrom(ip, 32) + return &prefix, IPv4, nil + case ip.Is6(): + prefix := netip.PrefixFrom(ip, 128) + return &prefix, IPv6, nil + default: + return nil, "", ErrInvalidIPLength + } + + case *net.IPNet: + prefix, ok := netipx.FromStdIPNet(src) + if !ok { + return nil, "", ErrInvalidIPNet + } + ip := prefix.Addr() + switch { + case ip.Is4(): + return &prefix, IPv4, nil + case ip.Is6(): + return &prefix, IPv6, nil + default: + return nil, "", ErrInvalidIPLength + } + + case netip.Addr: + switch { + case src.Is4(): + prefix := netip.PrefixFrom(src, 32) + return &prefix, IPv4, nil + case src.Is6(): + prefix := netip.PrefixFrom(src, 128) + return &prefix, IPv6, nil + default: + return nil, "", ErrInvalidIPLength + } + + case *netip.Addr: + switch { + case src.Is4(): + prefix := netip.PrefixFrom(*src, 32) + return &prefix, IPv4, nil + case src.Is6(): + prefix := netip.PrefixFrom(*src, 128) + return &prefix, IPv6, nil + default: + return nil, "", ErrInvalidIPLength + } + + case netip.Prefix: + ip := src.Addr() + switch { + case ip.Is4(): + return &src, IPv4, nil + case ip.Is6(): + return &src, IPv6, nil + default: + return nil, "", ErrInvalidIPLength + } + + case *netip.Prefix: + ip := src.Addr() + switch { + case ip.Is4(): + return src, IPv4, nil + case ip.Is6(): + return src, IPv6, nil + default: + return nil, "", ErrInvalidIPLength + } + + case string: + _, network, err := net.ParseCIDR(src) + switch err { + case nil: + prefix, ok := netipx.FromStdIPNet(network) + if !ok { + return nil, "", ErrInvalidIPNet + } + ip := prefix.Addr() + switch { + case ip.Is4(): + return &prefix, IPv4, nil + case ip.Is6(): + return &prefix, IPv6, nil + default: + return nil, "", ErrInvalidIPLength + } + + default: + ip, err := netip.ParseAddr(src) + if err != nil { + return nil, "", err + } + switch { + case ip.Is4(): + prefix := netip.PrefixFrom(ip, 32) + return &prefix, IPv4, nil + case ip.Is6(): + prefix := netip.PrefixFrom(ip, 128) + return &prefix, IPv6, nil + default: + return nil, "", ErrInvalidIPLength + } + } + } + + return nil, "", ErrInvalidPrefixType +} + +func (e *Entry) add(prefix *netip.Prefix, ipType IPType) error { + e.mu.Lock() + defer e.mu.Unlock() + + switch ipType { + case IPv4: + if !e.hasIPv4Builder() { + e.ipv4Builder = new(netipx.IPSetBuilder) + } + e.ipv4Builder.AddPrefix(*prefix) + case IPv6: + if !e.hasIPv6Builder() { + e.ipv6Builder = new(netipx.IPSetBuilder) + } + e.ipv6Builder.AddPrefix(*prefix) + default: + return ErrInvalidIPType + } + + return nil +} + +func (e *Entry) remove(prefix *netip.Prefix, ipType IPType) error { + e.mu.Lock() + defer e.mu.Unlock() + + switch ipType { + case IPv4: + if e.hasIPv4Builder() { + e.ipv4Builder.RemovePrefix(*prefix) + } + case IPv6: + if e.hasIPv6Builder() { + e.ipv6Builder.RemovePrefix(*prefix) + } + default: + return ErrInvalidIPType + } + + return nil +} + +func (e *Entry) AddPrefix(cidr any) error { + prefix, ipType, err := e.processPrefix(cidr) + if err != nil { + return err + } + if err := e.add(prefix, ipType); err != nil { + return err + } + return nil +} + +func (e *Entry) RemovePrefix(cidr string) error { + prefix, ipType, err := e.processPrefix(cidr) + if err != nil { + return err + } + if err := e.remove(prefix, ipType); err != nil { + return err + } + return nil +} + +func (e *Entry) MarshalText(opts ...IgnoreIPOption) ([]string, error) { + var ignoreIPType IPType + for _, opt := range opts { + if opt != nil { + ignoreIPType = opt() + } + } + disableIPv4, disableIPv6 := false, false + switch ignoreIPType { + case IPv4: + disableIPv4 = true + case IPv6: + disableIPv6 = true + } + + prefixSet := make([]string, 0, 1024) + + if !disableIPv4 && e.hasIPv4Builder() { + ipv4set, err := e.ipv4Builder.IPSet() + if err != nil { + return nil, err + } + prefixes := ipv4set.Prefixes() + for _, prefix := range prefixes { + prefixSet = append(prefixSet, prefix.String()) + } + } + + if !disableIPv6 && e.hasIPv6Builder() { + ipv6set, err := e.ipv6Builder.IPSet() + if err != nil { + return nil, err + } + prefixes := ipv6set.Prefixes() + for _, prefix := range prefixes { + prefixSet = append(prefixSet, prefix.String()) + } + } + + if len(prefixSet) > 0 { + return prefixSet, nil + } + + return nil, fmt.Errorf("entry %s has no prefix", e.GetName()) +} + +type IgnoreIPOption func() IPType + +func IgnoreIPv4() IPType { + return IPv4 +} + +func IgnoreIPv6() IPType { + return IPv6 +} + +type Container interface { + GetEntry(name string) (*Entry, bool) + Add(entry *Entry, opts ...IgnoreIPOption) error + Remove(name string, opts ...IgnoreIPOption) + Loop() <-chan *Entry +} + +type container struct { + entries *sync.Map // map[name]*Entry +} + +func NewContainer() Container { + return &container{ + entries: new(sync.Map), + } +} + +func (c *container) isValid() bool { + if c == nil || c.entries == nil { + return false + } + return true +} + +func (c *container) GetEntry(name string) (*Entry, bool) { + if !c.isValid() { + return nil, false + } + val, ok := c.entries.Load(strings.ToUpper(strings.TrimSpace(name))) + if !ok { + return nil, false + } + return val.(*Entry), true +} + +func (c *container) Loop() <-chan *Entry { + ch := make(chan *Entry, 300) + go func() { + c.entries.Range(func(key, value any) bool { + ch <- value.(*Entry) + return true + }) + close(ch) + }() + return ch +} + +func (c *container) Add(entry *Entry, opts ...IgnoreIPOption) error { + var ignoreIPType IPType + for _, opt := range opts { + if opt != nil { + ignoreIPType = opt() + } + } + + name := entry.GetName() + val, found := c.GetEntry(name) + switch found { + case true: + var ipv4set, ipv6set *netipx.IPSet + var err4, err6 error + if entry.hasIPv4Builder() { + ipv4set, err4 = entry.ipv4Builder.IPSet() + if err4 != nil { + return err4 + } + } + if entry.hasIPv6Builder() { + ipv6set, err6 = entry.ipv6Builder.IPSet() + if err6 != nil { + return err6 + } + } + switch ignoreIPType { + case IPv4: + if !val.hasIPv6Builder() { + val.ipv6Builder = new(netipx.IPSetBuilder) + } + val.ipv6Builder.AddSet(ipv6set) + case IPv6: + if !val.hasIPv4Builder() { + val.ipv4Builder = new(netipx.IPSetBuilder) + } + val.ipv4Builder.AddSet(ipv4set) + default: + if !val.hasIPv4Builder() { + val.ipv4Builder = new(netipx.IPSetBuilder) + } + if !val.hasIPv6Builder() { + val.ipv6Builder = new(netipx.IPSetBuilder) + } + val.ipv4Builder.AddSet(ipv4set) + val.ipv6Builder.AddSet(ipv6set) + } + c.entries.Store(name, val) + + case false: + switch ignoreIPType { + case IPv4: + entry.ipv4Builder = nil + case IPv6: + entry.ipv6Builder = nil + } + c.entries.Store(name, entry) + } + + return nil +} + +func (c *container) Remove(name string, opts ...IgnoreIPOption) { + val, found := c.GetEntry(name) + if !found { + log.Printf("failed to remove non-existent entry %s", name) + return + } + + var ignoreIPType IPType + for _, opt := range opts { + if opt != nil { + ignoreIPType = opt() + } + } + + switch ignoreIPType { + case IPv4: + val.ipv6Builder = nil + c.entries.Store(name, val) + case IPv6: + val.ipv4Builder = nil + c.entries.Store(name, val) + default: + c.entries.Delete(name) + } +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..7d954f1 --- /dev/null +++ b/main.go @@ -0,0 +1,150 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/v2fly/geoip/lib" + router "github.com/v2fly/v2ray-core/v5/app/router/routercommon" + "google.golang.org/protobuf/proto" +) + +var ( + // GeoIP flags + list = flag.Bool("l", false, "List all available input and output formats") + configFile = flag.String("c", "config.json", "Path to the config file") + + // Geosite flags + dataPath = flag.String("datapath", "./data", "Path to your custom 'data' directory") + outputName = flag.String("outputname", "dlc.dat", "Name of the generated dat file") + outputDir = flag.String("outputdir", "./", "Directory to place all generated files") + exportLists = flag.String("exportlists", "", "Lists to be flattened and exported in plaintext format, separated by ',' comma") + + mode = flag.String("m", "geoip", "Specify the mode to run: 'geoip' or 'geosite'") +) + +func main() { + flag.Parse() + + switch *mode { + case "geoip": + runGeoIP() + case "geosite": + runGeosite() + default: + log.Fatal("Unknown mode. Use 'geoip' or 'geosite'.") + } +} + +// GeoIP generation logic +func runGeoIP() { + if *list { + lib.ListInputConverter() + lib.ListOutputConverter() + return + } + + instance, err := lib.NewInstance() + if err != nil { + log.Fatal(err) + } + + if err := instance.Init(*configFile); err != nil { + log.Fatal(err) + } + + if err := instance.Run(); err != nil { + log.Fatal(err) + } +} + +// Geosite generation logic +func runGeosite() { + ref := make(map[string][]string) + + // Load and process all ref files + err := filepath.Walk(*dataPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if !info.IsDir() { + ref[info.Name()] = append(ref[info.Name()], path) + } + return nil + }) + if err != nil { + fmt.Println("Failed: ", err) + os.Exit(1) + } + + // Create output directory if not exist + if _, err := os.Stat(*outputDir); os.IsNotExist(err) { + if mkErr := os.MkdirAll(*outputDir, 0755); mkErr != nil { + fmt.Println("Failed: ", mkErr) + os.Exit(1) + } + } + + protoList := new(router.GeoSiteList) + var existList []string + for refName, list := range ref { + pl, err := ParseList(list, ref) + if err != nil { + fmt.Println("Failed: ", err) + os.Exit(1) + } + site, err := pl.toProto() + if err != nil { + fmt.Println("Failed: ", err) + os.Exit(1) + } + protoList.Entry = append(protoList.Entry, site) + + // Flatten and export plaintext list + if *exportLists != "" { + if existList != nil { + exportPlainTextList(existList, refName, pl) + } else { + exportedListSlice := strings.Split(*exportLists, ",") + for _, exportedListName := range exportedListSlice { + fileName := filepath.Join(*dataPath, exportedListName) + _, err := os.Stat(fileName) + if err == nil || os.IsExist(err) { + existList = append(existList, exportedListName) + } else { + fmt.Printf("'%s' list does not exist in '%s' directory.\n", exportedListName, *dataPath) + } + } + if existList != nil { + exportPlainTextList(existList, refName, pl) + } + } + } + } + + // Sort protoList so the marshaled list is reproducible + sort.SliceStable(protoList.Entry, func(i, j int) bool { + return protoList.Entry[i].CountryCode < protoList.Entry[j].CountryCode + }) + + protoBytes, err := proto.Marshal(protoList) + if err != nil { + fmt.Println("Failed:", err) + os.Exit(1) + } + if err := os.WriteFile(filepath.Join(*outputDir, *outputName), protoBytes, 0644); err != nil { + fmt.Println("Failed: ", err) + os.Exit(1) + } else { + fmt.Println(*outputName, "has been generated successfully.") + } +} + +// Additional utility functions for Geosite +// You may include ParseList, toProto, exportPlainTextList here if they are not yet defined in other packages. diff --git a/plugin/maxmind/country_csv.go b/plugin/maxmind/country_csv.go new file mode 100644 index 0000000..7773491 --- /dev/null +++ b/plugin/maxmind/country_csv.go @@ -0,0 +1,214 @@ +package maxmind + +import ( + "encoding/csv" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + + "github.com/v2fly/geoip/lib" +) + +const ( + typeCountryCSV = "maxmindGeoLite2CountryCSV" + descCountryCSV = "Convert MaxMind GeoLite2 country CSV data to other formats" +) + +var ( + defaultCCFile = filepath.Join("./", "geolite2", "GeoLite2-Country-Locations-en.csv") + defaultIPv4File = filepath.Join("./", "geolite2", "GeoLite2-Country-Blocks-IPv4.csv") + defaultIPv6File = filepath.Join("./", "geolite2", "GeoLite2-Country-Blocks-IPv6.csv") +) + +func init() { + lib.RegisterInputConfigCreator(typeCountryCSV, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { + return newGeoLite2CountryCSV(action, data) + }) + lib.RegisterInputConverter(typeCountryCSV, &geoLite2CountryCSV{ + Description: descCountryCSV, + }) +} + +func newGeoLite2CountryCSV(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { + var tmp struct { + CountryCodeFile string `json:"country"` + IPv4File string `json:"ipv4"` + IPv6File string `json:"ipv6"` + Want []string `json:"wantedList"` + OnlyIPType lib.IPType `json:"onlyIPType"` + } + + if len(data) > 0 { + if err := json.Unmarshal(data, &tmp); err != nil { + return nil, err + } + } + + if tmp.CountryCodeFile == "" { + tmp.CountryCodeFile = defaultCCFile + } + + if tmp.IPv4File == "" { + tmp.IPv4File = defaultIPv4File + } + + if tmp.IPv6File == "" { + tmp.IPv6File = defaultIPv6File + } + + return &geoLite2CountryCSV{ + Type: typeCountryCSV, + Action: action, + Description: descCountryCSV, + CountryCodeFile: tmp.CountryCodeFile, + IPv4File: tmp.IPv4File, + IPv6File: tmp.IPv6File, + Want: tmp.Want, + OnlyIPType: tmp.OnlyIPType, + }, nil +} + +type geoLite2CountryCSV struct { + Type string + Action lib.Action + Description string + CountryCodeFile string + IPv4File string + IPv6File string + Want []string + OnlyIPType lib.IPType +} + +func (g *geoLite2CountryCSV) GetType() string { + return g.Type +} + +func (g *geoLite2CountryCSV) GetAction() lib.Action { + return g.Action +} + +func (g *geoLite2CountryCSV) GetDescription() string { + return g.Description +} + +func (g *geoLite2CountryCSV) Input(container lib.Container) (lib.Container, error) { + ccMap, err := g.getCountryCode() + if err != nil { + return nil, err + } + + entries := make(map[string]*lib.Entry) + + if g.IPv4File != "" { + if err := g.process(g.IPv4File, ccMap, entries); err != nil { + return nil, err + } + } + + if g.IPv6File != "" { + if err := g.process(g.IPv6File, ccMap, entries); err != nil { + return nil, err + } + } + + var ignoreIPType lib.IgnoreIPOption + switch g.OnlyIPType { + case lib.IPv4: + ignoreIPType = lib.IgnoreIPv6 + case lib.IPv6: + ignoreIPType = lib.IgnoreIPv4 + } + + for name, entry := range entries { + switch g.Action { + case lib.ActionAdd: + if err := container.Add(entry, ignoreIPType); err != nil { + return nil, err + } + case lib.ActionRemove: + container.Remove(name, ignoreIPType) + default: + return nil, lib.ErrUnknownAction + } + } + + return container, nil +} + +func (g *geoLite2CountryCSV) getCountryCode() (map[string]string, error) { + ccReader, err := os.Open(g.CountryCodeFile) + if err != nil { + return nil, err + } + defer ccReader.Close() + + reader := csv.NewReader(ccReader) + lines, err := reader.ReadAll() + if err != nil { + return nil, err + } + + ccMap := make(map[string]string) + for _, line := range lines[1:] { + id := strings.TrimSpace(line[0]) + countryCode := strings.TrimSpace(line[4]) + if id == "" || countryCode == "" { + continue + } + ccMap[id] = strings.ToUpper(countryCode) + } + return ccMap, nil +} + +func (g *geoLite2CountryCSV) process(file string, ccMap map[string]string, entries map[string]*lib.Entry) error { + if len(ccMap) == 0 { + return errors.New("country code list must be specified") + } + if entries == nil { + entries = make(map[string]*lib.Entry) + } + + fReader, err := os.Open(file) + if err != nil { + return err + } + defer fReader.Close() + + reader := csv.NewReader(fReader) + lines, err := reader.ReadAll() + if err != nil { + return err + } + + // Filter want list + wantList := make(map[string]bool) + for _, want := range g.Want { + if want = strings.ToUpper(strings.TrimSpace(want)); want != "" { + wantList[want] = true + } + } + + for _, line := range lines[1:] { + ccID := strings.TrimSpace(line[1]) + if countryCode, found := ccMap[ccID]; found { + if len(wantList) > 0 { + if _, found := wantList[countryCode]; !found { + continue + } + } + cidrStr := strings.ToLower(strings.TrimSpace(line[0])) + entry, found := entries[countryCode] + if !found { + entry = lib.NewEntry(countryCode) + } + if err := entry.AddPrefix(cidrStr); err != nil { + return err + } + entries[countryCode] = entry + } + } + + return nil +} diff --git a/plugin/maxmind/mmdb_in.go b/plugin/maxmind/mmdb_in.go new file mode 100644 index 0000000..9261f18 --- /dev/null +++ b/plugin/maxmind/mmdb_in.go @@ -0,0 +1,227 @@ +package maxmind + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/oschwald/maxminddb-golang" + "github.com/v2fly/geoip/lib" +) + +const ( + typeMaxmindMMDBIn = "maxmindMMDB" + descMaxmindMMDBIn = "Convert MaxMind mmdb database to other formats" +) + +var ( + defaultMMDBFile = filepath.Join("./", "geolite2", "GeoLite2-Country.mmdb") + tempMMDBPath = filepath.Join("./", "tmp") + tempMMDBFile = filepath.Join(tempMMDBPath, "input.mmdb") +) + +func init() { + lib.RegisterInputConfigCreator(typeMaxmindMMDBIn, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { + return newMaxmindMMDBIn(action, data) + }) + lib.RegisterInputConverter(typeMaxmindMMDBIn, &maxmindMMDBIn{ + Description: descMaxmindMMDBIn, + }) +} + +func newMaxmindMMDBIn(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { + var tmp struct { + URI string `json:"uri"` + Want []string `json:"wantedList"` + OnlyIPType lib.IPType `json:"onlyIPType"` + } + + if len(data) > 0 { + if err := json.Unmarshal(data, &tmp); err != nil { + return nil, err + } + } + + if tmp.URI == "" { + tmp.URI = defaultMMDBFile + } + + return &maxmindMMDBIn{ + Type: typeMaxmindMMDBIn, + Action: action, + Description: descMaxmindMMDBIn, + URI: tmp.URI, + Want: tmp.Want, + OnlyIPType: tmp.OnlyIPType, + }, nil +} + +type maxmindMMDBIn struct { + Type string + Action lib.Action + Description string + URI string + Want []string + OnlyIPType lib.IPType +} + +func (g *maxmindMMDBIn) GetType() string { + return g.Type +} + +func (g *maxmindMMDBIn) GetAction() lib.Action { + return g.Action +} + +func (g *maxmindMMDBIn) GetDescription() string { + return g.Description +} + +func (g *maxmindMMDBIn) Input(container lib.Container) (lib.Container, error) { + var fd io.ReadCloser + var err error + switch { + case strings.HasPrefix(g.URI, "http://"), strings.HasPrefix(g.URI, "https://"): + fd, err = g.downloadFile(g.URI) + default: + fd, err = os.Open(g.URI) + } + + if err != nil { + return nil, err + } + + err = g.moveFile(fd) + if err != nil { + return nil, err + } + + entries := make(map[string]*lib.Entry) + err = g.generateEntries(entries) + if err != nil { + return nil, err + } + + if len(entries) == 0 { + return nil, fmt.Errorf("❌ [type %s | action %s] no entry is newly generated", typeMaxmindMMDBIn, g.Action) + } + + var ignoreIPType lib.IgnoreIPOption + switch g.OnlyIPType { + case lib.IPv4: + ignoreIPType = lib.IgnoreIPv6 + case lib.IPv6: + ignoreIPType = lib.IgnoreIPv4 + } + + // Filter want list + wantList := make(map[string]bool) + for _, want := range g.Want { + if want = strings.ToUpper(strings.TrimSpace(want)); want != "" { + wantList[want] = true + } + } + + for _, entry := range entries { + name := entry.GetName() + if len(wantList) > 0 && !wantList[name] { + continue + } + + switch g.Action { + case lib.ActionAdd: + if err := container.Add(entry, ignoreIPType); err != nil { + return nil, err + } + case lib.ActionRemove: + container.Remove(name, ignoreIPType) + } + } + + return container, nil +} + +func (g *maxmindMMDBIn) downloadFile(url string) (io.ReadCloser, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("failed to get remote file %s, http status code %d", url, resp.StatusCode) + } + + return resp.Body, nil +} + +func (g *maxmindMMDBIn) moveFile(src io.ReadCloser) error { + defer src.Close() + + err := os.MkdirAll(tempMMDBPath, 0755) + if err != nil { + return err + } + + out, err := os.Create(tempMMDBFile) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, src) + + return err +} + +func (g *maxmindMMDBIn) generateEntries(entries map[string]*lib.Entry) error { + db, err := maxminddb.Open(tempMMDBFile) + if err != nil { + return err + } + defer db.Close() + + record := struct { + Country struct { + IsoCode string `maxminddb:"iso_code"` + } `maxminddb:"country"` + }{} + + networks := db.Networks(maxminddb.SkipAliasedNetworks) + for networks.Next() { + subnet, err := networks.Network(&record) + if err != nil { + continue + } + + var entry *lib.Entry + name := strings.ToUpper(record.Country.IsoCode) + if theEntry, found := entries[name]; found { + entry = theEntry + } else { + entry = lib.NewEntry(name) + } + + switch g.Action { + case lib.ActionAdd: + if err := entry.AddPrefix(subnet); err != nil { + return err + } + case lib.ActionRemove: + if err := entry.RemovePrefix(subnet.String()); err != nil { + return err + } + } + + entries[name] = entry + } + + if networks.Err() != nil { + return networks.Err() + } + + return nil +} diff --git a/plugin/plaintext/text_in.go b/plugin/plaintext/text_in.go new file mode 100644 index 0000000..f3fd7ed --- /dev/null +++ b/plugin/plaintext/text_in.go @@ -0,0 +1,227 @@ +package plaintext + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/v2fly/geoip/lib" +) + +const ( + typeTextIn = "text" + descTextIn = "Convert plaintext IP and CIDR to other formats" +) + +func init() { + lib.RegisterInputConfigCreator(typeTextIn, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { + return newTextIn(action, data) + }) + lib.RegisterInputConverter(typeTextIn, &textIn{ + Description: descTextIn, + }) +} + +func newTextIn(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { + var tmp struct { + Name string `json:"name"` + URI string `json:"uri"` + InputDir string `json:"inputDir"` + OnlyIPType lib.IPType `json:"onlyIPType"` + } + + if len(data) > 0 { + if err := json.Unmarshal(data, &tmp); err != nil { + return nil, err + } + } + + if tmp.Name == "" && tmp.URI == "" && tmp.InputDir == "" { + return nil, fmt.Errorf("type %s | action %s missing inputdir or name or uri", typeTextIn, action) + } + + if (tmp.Name != "" && tmp.URI == "") || (tmp.Name == "" && tmp.URI != "") { + return nil, fmt.Errorf("type %s | action %s name & uri must be specified together", typeTextIn, action) + } + + return &textIn{ + Type: typeTextIn, + Action: action, + Description: descTextIn, + Name: tmp.Name, + URI: tmp.URI, + InputDir: tmp.InputDir, + OnlyIPType: tmp.OnlyIPType, + }, nil +} + +type textIn struct { + Type string + Action lib.Action + Description string + Name string + URI string + InputDir string + OnlyIPType lib.IPType +} + +func (t *textIn) GetType() string { + return t.Type +} + +func (t *textIn) GetAction() lib.Action { + return t.Action +} + +func (t *textIn) GetDescription() string { + return t.Description +} + +func (t *textIn) Input(container lib.Container) (lib.Container, error) { + entries := make(map[string]*lib.Entry) + var err error + + switch { + case t.InputDir != "": + err = t.walkDir(t.InputDir, entries) + case t.Name != "" && t.URI != "": + switch { + case strings.HasPrefix(t.URI, "http://"), strings.HasPrefix(t.URI, "https://"): + err = t.walkRemoteFile(t.URI, t.Name, entries) + default: + err = t.walkLocalFile(t.URI, t.Name, entries) + } + default: + return nil, fmt.Errorf("config missing argument inputDir or name or uri") + } + + if err != nil { + return nil, err + } + + var ignoreIPType lib.IgnoreIPOption + switch t.OnlyIPType { + case lib.IPv4: + ignoreIPType = lib.IgnoreIPv6 + case lib.IPv6: + ignoreIPType = lib.IgnoreIPv4 + } + + if len(entries) == 0 { + return nil, fmt.Errorf("type %s | action %s no entry are generated", t.Type, t.Action) + } + + for _, entry := range entries { + switch t.Action { + case lib.ActionAdd: + if err := container.Add(entry, ignoreIPType); err != nil { + return nil, err + } + case lib.ActionRemove: + container.Remove(entry.GetName(), ignoreIPType) + } + } + + return container, nil +} + +func (t *textIn) walkDir(dir string, entries map[string]*lib.Entry) error { + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + if err := t.walkLocalFile(path, "", entries); err != nil { + return err + } + + return nil + }) + + return err +} + +func (t *textIn) walkLocalFile(path, name string, entries map[string]*lib.Entry) error { + name = strings.TrimSpace(name) + var filename string + if name != "" { + filename = name + } else { + filename = filepath.Base(path) + } + + // check filename + if !regexp.MustCompile(`^[a-zA-Z0-9_.\-]+$`).MatchString(filename) { + return fmt.Errorf("filename %s cannot be entry name, please remove special characters in it", filename) + } + dotIndex := strings.LastIndex(filename, ".") + if dotIndex > 0 { + filename = filename[:dotIndex] + } + + if _, found := entries[filename]; found { + return fmt.Errorf("found duplicated file %s", filename) + } + + entry := lib.NewEntry(filename) + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + if err := t.scanFile(file, entry); err != nil { + return err + } + + entries[filename] = entry + + return nil +} + +func (t *textIn) walkRemoteFile(url, name string, entries map[string]*lib.Entry) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fmt.Errorf("failed to get remote file %s, http status code %d", url, resp.StatusCode) + } + + entry := lib.NewEntry(name) + if err := t.scanFile(resp.Body, entry); err != nil { + return err + } + + entries[name] = entry + return nil +} + +func (t *textIn) scanFile(reader io.Reader, entry *lib.Entry) error { + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + if err := entry.AddPrefix(line); err != nil { + return err + } + } + + if err := scanner.Err(); err != nil { + return err + } + + return nil +} diff --git a/plugin/plaintext/text_out.go b/plugin/plaintext/text_out.go new file mode 100644 index 0000000..68f1d92 --- /dev/null +++ b/plugin/plaintext/text_out.go @@ -0,0 +1,166 @@ +package plaintext + +import ( + "bytes" + "encoding/json" + "log" + "os" + "path/filepath" + "strings" + + "github.com/v2fly/geoip/lib" +) + +const ( + typeTextOut = "text" + descTextOut = "Convert data to plaintext CIDR format" +) + +var ( + defaultOutputDir = filepath.Join("./", "output", "text") +) + +func init() { + lib.RegisterOutputConfigCreator(typeTextOut, func(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) { + return newTextOut(action, data) + }) + lib.RegisterOutputConverter(typeTextOut, &textOut{ + Description: descTextOut, + }) +} + +func newTextOut(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) { + var tmp struct { + OutputDir string `json:"outputDir"` + Want []string `json:"wantedList"` + OnlyIPType lib.IPType `json:"onlyIPType"` + } + + if len(data) > 0 { + if err := json.Unmarshal(data, &tmp); err != nil { + return nil, err + } + } + + if tmp.OutputDir == "" { + tmp.OutputDir = defaultOutputDir + } + + return &textOut{ + Type: typeTextOut, + Action: action, + Description: descTextOut, + OutputDir: tmp.OutputDir, + Want: tmp.Want, + OnlyIPType: tmp.OnlyIPType, + }, nil +} + +type textOut struct { + Type string + Action lib.Action + Description string + OutputDir string + Want []string + OnlyIPType lib.IPType +} + +func (t *textOut) GetType() string { + return t.Type +} + +func (t *textOut) GetAction() lib.Action { + return t.Action +} + +func (t *textOut) GetDescription() string { + return t.Description +} + +func (t *textOut) Output(container lib.Container) error { + // Filter want list + wantList := make(map[string]bool) + for _, want := range t.Want { + if want = strings.ToUpper(strings.TrimSpace(want)); want != "" { + wantList[want] = true + } + } + + switch len(wantList) { + case 0: + for entry := range container.Loop() { + cidrList, err := t.marshalText(entry) + if err != nil { + return err + } + filename := strings.ToLower(entry.GetName()) + ".txt" + if err := t.writeFile(filename, cidrList); err != nil { + return err + } + } + + default: + for name := range wantList { + entry, found := container.GetEntry(name) + if !found { + log.Printf("❌ entry %s not found", name) + continue + } + cidrList, err := t.marshalText(entry) + if err != nil { + return err + } + filename := strings.ToLower(entry.GetName()) + ".txt" + if err := t.writeFile(filename, cidrList); err != nil { + return err + } + } + } + + return nil +} + +func (t *textOut) marshalText(entry *lib.Entry) ([]string, error) { + var entryCidr []string + var err error + switch t.OnlyIPType { + case lib.IPv4: + entryCidr, err = entry.MarshalText(lib.IgnoreIPv6) + if err != nil { + return nil, err + } + case lib.IPv6: + entryCidr, err = entry.MarshalText(lib.IgnoreIPv4) + if err != nil { + return nil, err + } + default: + entryCidr, err = entry.MarshalText() + if err != nil { + return nil, err + } + } + + return entryCidr, nil +} + +func (t *textOut) writeFile(filename string, cidrList []string) error { + var buf bytes.Buffer + for _, cidr := range cidrList { + buf.WriteString(cidr) + buf.WriteString("\n") + } + cidrBytes := buf.Bytes() + + if err := os.MkdirAll(t.OutputDir, 0755); err != nil { + return err + } + + if err := os.WriteFile(filepath.Join(t.OutputDir, filename), cidrBytes, 0644); err != nil { + return err + } + + log.Printf("✅ [%s] %s --> %s", t.Type, filename, t.OutputDir) + + return nil +} diff --git a/plugin/special/cutter.go b/plugin/special/cutter.go new file mode 100644 index 0000000..8a8fdba --- /dev/null +++ b/plugin/special/cutter.go @@ -0,0 +1,96 @@ +package special + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/v2fly/geoip/lib" +) + +const ( + typeCutter = "cutter" + descCutter = "Remove data from previous steps" +) + +func init() { + lib.RegisterInputConfigCreator(typeCutter, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { + return newCutter(action, data) + }) + lib.RegisterInputConverter(typeCutter, &cutter{ + Description: descCutter, + }) +} + +func newCutter(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { + var tmp struct { + Want []string `json:"wantedList"` + OnlyIPType lib.IPType `json:"onlyIPType"` + } + + if len(data) > 0 { + if err := json.Unmarshal(data, &tmp); err != nil { + return nil, err + } + } + + if action != lib.ActionRemove { + return nil, fmt.Errorf("type %s only supports `remove` action", typeCutter) + } + + return &cutter{ + Type: typeCutter, + Action: action, + Description: descCutter, + Want: tmp.Want, + OnlyIPType: tmp.OnlyIPType, + }, nil +} + +type cutter struct { + Type string + Action lib.Action + Description string + Want []string + OnlyIPType lib.IPType +} + +func (c *cutter) GetType() string { + return c.Type +} + +func (c *cutter) GetAction() lib.Action { + return c.Action +} + +func (c *cutter) GetDescription() string { + return c.Description +} + +func (c *cutter) Input(container lib.Container) (lib.Container, error) { + var ignoreIPType lib.IgnoreIPOption + switch c.OnlyIPType { + case lib.IPv4: + ignoreIPType = lib.IgnoreIPv6 + case lib.IPv6: + ignoreIPType = lib.IgnoreIPv4 + } + + // Filter want list + wantList := make(map[string]bool) + for _, want := range c.Want { + if want = strings.ToUpper(strings.TrimSpace(want)); want != "" { + wantList[want] = true + } + } + + for entry := range container.Loop() { + name := entry.GetName() + if len(wantList) > 0 && !wantList[name] { + continue + } + container.Remove(name, ignoreIPType) + } + + return container, nil +} diff --git a/plugin/special/private.go b/plugin/special/private.go new file mode 100644 index 0000000..e9dceca --- /dev/null +++ b/plugin/special/private.go @@ -0,0 +1,94 @@ +package special + +import ( + "encoding/json" + + "github.com/v2fly/geoip/lib" +) + +const ( + entryNamePrivate = "private" + typePrivate = "private" + descPrivate = "Convert LAN and private network CIDR to other formats" +) + +var privateCIDRs = []string{ + "0.0.0.0/8", + "10.0.0.0/8", + "100.64.0.0/10", + "127.0.0.0/8", + "169.254.0.0/16", + "172.16.0.0/12", + "192.0.0.0/24", + "192.0.2.0/24", + "192.88.99.0/24", + "192.168.0.0/16", + "198.18.0.0/15", + "198.51.100.0/24", + "203.0.113.0/24", + "224.0.0.0/4", + "240.0.0.0/4", + "255.255.255.255/32", + "::/128", + "::1/128", + "fc00::/7", + "fe80::/10", + "ff00::/8", +} + +func init() { + lib.RegisterInputConfigCreator(typePrivate, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { + return newPrivate(action, data) + }) + lib.RegisterInputConverter(typePrivate, &private{ + Description: descPrivate, + }) +} + +func newPrivate(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { + return &private{ + Type: typePrivate, + Action: action, + Description: descPrivate, + }, nil +} + +type private struct { + Type string + Action lib.Action + Description string +} + +func (p *private) GetType() string { + return p.Type +} + +func (p *private) GetAction() lib.Action { + return p.Action +} + +func (p *private) GetDescription() string { + return p.Description +} + +func (p *private) Input(container lib.Container) (lib.Container, error) { + entry := lib.NewEntry(entryNamePrivate) + for _, cidr := range privateCIDRs { + if err := entry.AddPrefix(cidr); err != nil { + return nil, err + } + } + + switch p.Action { + case lib.ActionAdd: + if err := container.Add(entry); err != nil { + return nil, err + } + case lib.ActionRemove: + container.Remove(entryNamePrivate) + default: + return nil, lib.ErrUnknownAction + } + + return container, nil +} diff --git a/plugin/special/test.go b/plugin/special/test.go new file mode 100644 index 0000000..1a48893 --- /dev/null +++ b/plugin/special/test.go @@ -0,0 +1,74 @@ +package special + +import ( + "encoding/json" + + "github.com/v2fly/geoip/lib" +) + +const ( + entryNameTest = "test" + typeTest = "test" + descTest = "Convert specific CIDR to other formats (for test only)" +) + +var testCIDRs = []string{ + "127.0.0.0/8", +} + +func init() { + lib.RegisterInputConfigCreator(typeTest, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { + return newTest(action, data) + }) + lib.RegisterInputConverter(typeTest, &test{ + Description: descTest, + }) +} + +func newTest(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { + return &test{ + Type: typeTest, + Action: action, + Description: descTest, + }, nil +} + +type test struct { + Type string + Action lib.Action + Description string +} + +func (t *test) GetType() string { + return t.Type +} + +func (t *test) GetAction() lib.Action { + return t.Action +} + +func (t *test) GetDescription() string { + return t.Description +} + +func (t *test) Input(container lib.Container) (lib.Container, error) { + entry := lib.NewEntry(entryNameTest) + for _, cidr := range testCIDRs { + if err := entry.AddPrefix(cidr); err != nil { + return nil, err + } + } + + switch t.Action { + case lib.ActionAdd: + if err := container.Add(entry); err != nil { + return nil, err + } + case lib.ActionRemove: + container.Remove(entryNameTest) + default: + return nil, lib.ErrUnknownAction + } + + return container, nil +} diff --git a/plugin/v2ray/dat_in.go b/plugin/v2ray/dat_in.go new file mode 100644 index 0000000..03874a7 --- /dev/null +++ b/plugin/v2ray/dat_in.go @@ -0,0 +1,204 @@ +package v2ray + +import ( + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "os" + "strings" + + "github.com/v2fly/geoip/lib" + router "github.com/v2fly/v2ray-core/v5/app/router/routercommon" + "google.golang.org/protobuf/proto" +) + +const ( + typeGeoIPdatIn = "v2rayGeoIPDat" + descGeoIPdatIn = "Convert V2Ray GeoIP dat to other formats" +) + +func init() { + lib.RegisterInputConfigCreator(typeGeoIPdatIn, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { + return newGeoIPDatIn(action, data) + }) + lib.RegisterInputConverter(typeGeoIPdatIn, &geoIPDatIn{ + Description: descGeoIPdatIn, + }) +} + +func newGeoIPDatIn(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { + var tmp struct { + URI string `json:"uri"` + Want []string `json:"wantedList"` + OnlyIPType lib.IPType `json:"onlyIPType"` + } + + if len(data) > 0 { + if err := json.Unmarshal(data, &tmp); err != nil { + return nil, err + } + } + + if tmp.URI == "" { + return nil, fmt.Errorf("[type %s | action %s] uri must be specified in config", typeGeoIPdatIn, action) + } + + return &geoIPDatIn{ + Type: typeGeoIPdatIn, + Action: action, + Description: descGeoIPdatIn, + URI: tmp.URI, + Want: tmp.Want, + OnlyIPType: tmp.OnlyIPType, + }, nil +} + +type geoIPDatIn struct { + Type string + Action lib.Action + Description string + URI string + Want []string + OnlyIPType lib.IPType +} + +func (g *geoIPDatIn) GetType() string { + return g.Type +} + +func (g *geoIPDatIn) GetAction() lib.Action { + return g.Action +} + +func (g *geoIPDatIn) GetDescription() string { + return g.Description +} + +func (g *geoIPDatIn) Input(container lib.Container) (lib.Container, error) { + entries := make(map[string]*lib.Entry) + var err error + + switch { + case strings.HasPrefix(g.URI, "http://"), strings.HasPrefix(g.URI, "https://"): + err = g.walkRemoteFile(g.URI, entries) + default: + err = g.walkLocalFile(g.URI, entries) + } + + if err != nil { + return nil, err + } + + if len(entries) == 0 { + return nil, fmt.Errorf("❌ [type %s | action %s] no entry is newly generated", typeGeoIPdatIn, g.Action) + } + + var ignoreIPType lib.IgnoreIPOption + switch g.OnlyIPType { + case lib.IPv4: + ignoreIPType = lib.IgnoreIPv6 + case lib.IPv6: + ignoreIPType = lib.IgnoreIPv4 + } + + // Filter want list + wantList := make(map[string]bool) + for _, want := range g.Want { + if want = strings.ToUpper(strings.TrimSpace(want)); want != "" { + wantList[want] = true + } + } + + for _, entry := range entries { + name := entry.GetName() + if len(wantList) > 0 && !wantList[name] { + continue + } + + switch g.Action { + case lib.ActionAdd: + if err := container.Add(entry, ignoreIPType); err != nil { + return nil, err + } + case lib.ActionRemove: + container.Remove(name, ignoreIPType) + } + } + + return container, nil +} + +func (g *geoIPDatIn) walkLocalFile(path string, entries map[string]*lib.Entry) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + if err := g.generateEntries(file, entries); err != nil { + return err + } + + return nil +} + +func (g *geoIPDatIn) walkRemoteFile(url string, entries map[string]*lib.Entry) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fmt.Errorf("failed to get remote file %s, http status code %d", url, resp.StatusCode) + } + + if err := g.generateEntries(resp.Body, entries); err != nil { + return err + } + + return nil +} + +func (g *geoIPDatIn) generateEntries(reader io.Reader, entries map[string]*lib.Entry) error { + geoipBytes, err := io.ReadAll(reader) + if err != nil { + return err + } + + var geoipList router.GeoIPList + if err := proto.Unmarshal(geoipBytes, &geoipList); err != nil { + return err + } + + for _, geoip := range geoipList.Entry { + var entry *lib.Entry + name := geoip.CountryCode + if theEntry, found := entries[name]; found { + fmt.Printf("⚠️ [type %s | action %s] found duplicated entry: %s. Process anyway\n", typeGeoIPdatIn, g.Action, name) + entry = theEntry + } else { + entry = lib.NewEntry(name) + } + + for _, v2rayCIDR := range geoip.Cidr { + ipStr := net.IP(v2rayCIDR.GetIp()).String() + "/" + fmt.Sprint(v2rayCIDR.GetPrefix()) + switch g.Action { + case lib.ActionAdd: + if err := entry.AddPrefix(ipStr); err != nil { + return err + } + case lib.ActionRemove: + if err := entry.RemovePrefix(ipStr); err != nil { + return err + } + } + } + + entries[name] = entry + } + + return nil +} diff --git a/plugin/v2ray/dat_out.go b/plugin/v2ray/dat_out.go new file mode 100644 index 0000000..3c4ba23 --- /dev/null +++ b/plugin/v2ray/dat_out.go @@ -0,0 +1,227 @@ +package v2ray + +import ( + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/v2fly/geoip/lib" + router "github.com/v2fly/v2ray-core/v5/app/router/routercommon" + "github.com/v2fly/v2ray-core/v5/infra/conf/rule" + "google.golang.org/protobuf/proto" +) + +const ( + typeGeoIPdatOut = "v2rayGeoIPDat" + descGeoIPdatOut = "Convert data to V2Ray GeoIP dat format" +) + +var ( + defaultOutputName = "geoip.dat" + defaultOutputDir = filepath.Join("./", "output", "dat") +) + +func init() { + lib.RegisterOutputConfigCreator(typeGeoIPdatOut, func(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) { + return newGeoIPDat(action, data) + }) + lib.RegisterOutputConverter(typeGeoIPdatOut, &geoIPDatOut{ + Description: descGeoIPdatOut, + }) +} + +func newGeoIPDat(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) { + var tmp struct { + OutputName string `json:"outputName"` + OutputDir string `json:"outputDir"` + Want []string `json:"wantedList"` + OneFilePerList bool `json:"oneFilePerList"` + OnlyIPType lib.IPType `json:"onlyIPType"` + } + + if len(data) > 0 { + if err := json.Unmarshal(data, &tmp); err != nil { + return nil, err + } + } + + if tmp.OutputName == "" { + tmp.OutputName = defaultOutputName + } + + if tmp.OutputDir == "" { + tmp.OutputDir = defaultOutputDir + } + + return &geoIPDatOut{ + Type: typeGeoIPdatOut, + Action: action, + Description: descGeoIPdatOut, + OutputName: tmp.OutputName, + OutputDir: tmp.OutputDir, + Want: tmp.Want, + OneFilePerList: tmp.OneFilePerList, + OnlyIPType: tmp.OnlyIPType, + }, nil +} + +type geoIPDatOut struct { + Type string + Action lib.Action + Description string + OutputName string + OutputDir string + Want []string + OneFilePerList bool + OnlyIPType lib.IPType +} + +func (g *geoIPDatOut) GetType() string { + return g.Type +} + +func (g *geoIPDatOut) GetAction() lib.Action { + return g.Action +} + +func (g *geoIPDatOut) GetDescription() string { + return g.Description +} + +func (g *geoIPDatOut) Output(container lib.Container) error { + // Filter want list + wantList := make(map[string]bool) + for _, want := range g.Want { + if want = strings.ToUpper(strings.TrimSpace(want)); want != "" { + wantList[want] = true + } + } + + geoIPList := new(router.GeoIPList) + geoIPList.Entry = make([]*router.GeoIP, 0, 300) + updated := false + switch len(wantList) { + case 0: + for entry := range container.Loop() { + geoIP, err := g.generateGeoIP(entry) + if err != nil { + return err + } + geoIPList.Entry = append(geoIPList.Entry, geoIP) + updated = true + + if g.OneFilePerList { + geoIPBytes, err := proto.Marshal(geoIPList) + if err != nil { + return err + } + filename := strings.ToLower(entry.GetName()) + ".dat" + if err := g.writeFile(filename, geoIPBytes); err != nil { + return err + } + geoIPList.Entry = nil + } + } + + default: + for name := range wantList { + entry, found := container.GetEntry(name) + if !found { + log.Printf("❌ entry %s not found", name) + continue + } + geoIP, err := g.generateGeoIP(entry) + if err != nil { + return err + } + geoIPList.Entry = append(geoIPList.Entry, geoIP) + updated = true + + if g.OneFilePerList { + geoIPBytes, err := proto.Marshal(geoIPList) + if err != nil { + return err + } + filename := strings.ToLower(entry.GetName()) + ".dat" + if err := g.writeFile(filename, geoIPBytes); err != nil { + return err + } + geoIPList.Entry = nil + } + } + } + + // Sort to make reproducible builds + g.sort(geoIPList) + + if !g.OneFilePerList && updated { + geoIPBytes, err := proto.Marshal(geoIPList) + if err != nil { + return err + } + if err := g.writeFile(g.OutputName, geoIPBytes); err != nil { + return err + } + } + + return nil +} + +func (g *geoIPDatOut) generateGeoIP(entry *lib.Entry) (*router.GeoIP, error) { + var entryCidr []string + var err error + switch g.OnlyIPType { + case lib.IPv4: + entryCidr, err = entry.MarshalText(lib.IgnoreIPv6) + case lib.IPv6: + entryCidr, err = entry.MarshalText(lib.IgnoreIPv4) + default: + entryCidr, err = entry.MarshalText() + } + if err != nil { + return nil, err + } + + v2rayCIDR := make([]*router.CIDR, 0, 1024) + for _, cidrStr := range entryCidr { + cidr, err := rule.ParseIP(cidrStr) + if err != nil { + return nil, err + } + v2rayCIDR = append(v2rayCIDR, cidr) + } + + if len(v2rayCIDR) > 0 { + return &router.GeoIP{ + CountryCode: entry.GetName(), + Cidr: v2rayCIDR, + }, nil + } + + return nil, fmt.Errorf("entry %s has no CIDR", entry.GetName()) +} + +// Sort by country code to make reproducible builds +func (g *geoIPDatOut) sort(list *router.GeoIPList) { + sort.SliceStable(list.Entry, func(i, j int) bool { + return list.Entry[i].CountryCode < list.Entry[j].CountryCode + }) +} + +func (g *geoIPDatOut) writeFile(filename string, geoIPBytes []byte) error { + if err := os.MkdirAll(g.OutputDir, 0755); err != nil { + return err + } + + if err := os.WriteFile(filepath.Join(g.OutputDir, filename), geoIPBytes, 0644); err != nil { + return err + } + + log.Printf("✅ [%s] %s --> %s", g.Type, filename, g.OutputDir) + + return nil +}