diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index dff1218..0000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,24 +0,0 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates - -version: 2 -updates: - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "daily" - timezone: "Europe/Moscow" - time: "04:00" - pull-request-branch-name: - separator: "-" - - - package-ecosystem: "gomod" - directory: "/" - schedule: - interval: "daily" - timezone: "Europe/Moscow" - time: "04:00" - pull-request-branch-name: - separator: "-" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index f0f1389..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,101 +0,0 @@ -name: Build geoip.dat and geosite.dat - -on: - workflow_dispatch: - schedule: - - cron: "0 0 * * 0" - push: - branches: - - main - paths-ignore: - - "README.md" - - ".gitignore" - - "LICENSE" - - "**/dependabot.yml" - - "**/README.md" - -jobs: - build_geoip: - name: Build geoip.dat - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: "1.19" - - - name: Set variables for geoip - run: | - echo "TAG_NAME=$(date +%Y%m%d%H%M)" >> $GITHUB_ENV - echo "RELEASE_NAME=$(date +%Y%m%d%H%M)" >> $GITHUB_ENV - shell: bash - - - name: Build and run geoip.dat - run: | - go build -o geoip_gen main.go - ./geoip_gen -m geoip - - - name: Generate sha256 checksum for geoip.dat files - run: | - cd ./output/dat || exit 1 - for name in $(ls *.dat); do - sha256sum ${name} > ./${name}.sha256sum - done - - - name: Move geoip files to publish directory - run: | - mkdir -p publish - mv ./output/dat/*.dat ./output/dat/*.sha256sum ./publish - - - name: Release and upload geoip assets - run: | - gh release create ${{ env.TAG_NAME }} -t ${{ env.RELEASE_NAME }} \ - ./publish/*.dat \ - ./publish/*.dat.sha256sum - env: - GITHUB_TOKEN: ${{ github.token }} - - build_geosite: - name: Build geosite.dat - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: "1.19" - - - name: Set variables for geosite - run: | - echo "RELEASE_NAME=$(date +%Y-%m-%d)" >> $GITHUB_ENV - echo "TAG_NAME=$(date +%Y-%m-%d)" >> $GITHUB_ENV - shell: bash - - - name: Use local domains file - run: mkdir -p data && cp ./domains_all.lst ./data/refilter - - - name: Build and run geosite.dat - run: | - go build -o geosite_gen main.go - ./geosite_gen -m geosite --exportlists=refilter --outputname=geosite.dat - - - name: Generate sha256 hash for geosite.dat - run: sha256sum geosite.dat > geosite.dat.sha256sum - - - name: Move geosite files to publish directory - run: | - mkdir -p publish - mv ./*.dat ./*.sha256sum ./publish - - - name: Release and upload geosite assets - run: | - gh release create ${{ env.TAG_NAME }} -t ${{ env.RELEASE_NAME }} \ - ./publish/*.dat \ - ./publish/*.dat.sha256sum - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/go.mod b/go.mod deleted file mode 100644 index 642689e..0000000 --- a/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/1andrevich/Re-filter-lists - -go 1.23 diff --git a/lib/config.go b/lib/config.go deleted file mode 100644 index 20e5137..0000000 --- a/lib/config.go +++ /dev/null @@ -1,128 +0,0 @@ -package lib - -import ( - "encoding/json" - "errors" - "fmt" - "strings" -) - -var ( - inputConfigCreatorCache = make(map[string]inputConfigCreator) - outputConfigCreatorCache = make(map[string]outputConfigCreator) -) - -type inputConfigCreator func(Action, json.RawMessage) (InputConverter, error) - -type outputConfigCreator func(Action, json.RawMessage) (OutputConverter, error) - -func RegisterInputConfigCreator(id string, fn inputConfigCreator) error { - id = strings.ToLower(id) - if _, found := inputConfigCreatorCache[id]; found { - return errors.New("config creator has already been registered") - } - inputConfigCreatorCache[id] = fn - return nil -} - -func createInputConfig(id string, action Action, data json.RawMessage) (InputConverter, error) { - id = strings.ToLower(id) - fn, found := inputConfigCreatorCache[id] - if !found { - return nil, errors.New("unknown config type") - } - return fn(action, data) -} - -func RegisterOutputConfigCreator(id string, fn outputConfigCreator) error { - id = strings.ToLower(id) - if _, found := outputConfigCreatorCache[id]; found { - return errors.New("config creator has already been registered") - } - outputConfigCreatorCache[id] = fn - return nil -} - -func createOutputConfig(id string, action Action, data json.RawMessage) (OutputConverter, error) { - id = strings.ToLower(id) - fn, found := outputConfigCreatorCache[id] - if !found { - return nil, errors.New("unknown config type") - } - return fn(action, data) -} - -type config struct { - Input []*inputConvConfig `json:"input"` - Output []*outputConvConfig `json:"output"` -} - -type inputConvConfig struct { - iType string - action Action - converter InputConverter -} - -func (i *inputConvConfig) UnmarshalJSON(data []byte) error { - var temp struct { - Type string `json:"type"` - Action Action `json:"action"` - Args json.RawMessage `json:"args"` - } - - if err := json.Unmarshal(data, &temp); err != nil { - return err - } - - if !ActionsRegistry[temp.Action] { - return fmt.Errorf("invalid action %s in type %s", temp.Action, temp.Type) - } - - config, err := createInputConfig(temp.Type, temp.Action, temp.Args) - if err != nil { - return err - } - - i.iType = config.GetType() - i.action = config.GetAction() - i.converter = config - - return nil -} - -type outputConvConfig struct { - iType string - action Action - converter OutputConverter -} - -func (i *outputConvConfig) UnmarshalJSON(data []byte) error { - var temp struct { - Type string `json:"type"` - Action Action `json:"action"` - Args json.RawMessage `json:"args"` - } - - if err := json.Unmarshal(data, &temp); err != nil { - return err - } - - if temp.Action == "" { - temp.Action = ActionOutput - } - - if !ActionsRegistry[temp.Action] { - return fmt.Errorf("invalid action %s in type %s", temp.Action, temp.Type) - } - - config, err := createOutputConfig(temp.Type, temp.Action, temp.Args) - if err != nil { - return err - } - - i.iType = config.GetType() - i.action = config.GetAction() - i.converter = config - - return nil -} diff --git a/lib/error.go b/lib/error.go deleted file mode 100644 index 5778e96..0000000 --- a/lib/error.go +++ /dev/null @@ -1,13 +0,0 @@ -package lib - -import "errors" - -var ( - ErrDuplicatedConverter = errors.New("duplicated converter") - ErrUnknownAction = errors.New("unknown action") - ErrInvalidIPType = errors.New("invalid IP type") - ErrInvalidIP = errors.New("invalid IP address") - ErrInvalidIPLength = errors.New("invalid IP address length") - ErrInvalidIPNet = errors.New("invalid IPNet address") - ErrInvalidPrefixType = errors.New("invalid prefix type") -) diff --git a/lib/func.go b/lib/func.go deleted file mode 100644 index 8cef3ce..0000000 --- a/lib/func.go +++ /dev/null @@ -1,43 +0,0 @@ -package lib - -import ( - "fmt" - "strings" -) - -var ( - inputConverterMap = make(map[string]InputConverter) - outputConverterMap = make(map[string]OutputConverter) -) - -func ListInputConverter() { - fmt.Println("All available input formats:") - for name, ic := range inputConverterMap { - fmt.Printf(" - %s (%s)\n", name, ic.GetDescription()) - } -} - -func RegisterInputConverter(name string, c InputConverter) error { - name = strings.TrimSpace(name) - if _, ok := inputConverterMap[name]; ok { - return ErrDuplicatedConverter - } - inputConverterMap[name] = c - return nil -} - -func ListOutputConverter() { - fmt.Println("All available output formats:") - for name, oc := range outputConverterMap { - fmt.Printf(" - %s (%s)\n", name, oc.GetDescription()) - } -} - -func RegisterOutputConverter(name string, c OutputConverter) error { - name = strings.TrimSpace(name) - if _, ok := outputConverterMap[name]; ok { - return ErrDuplicatedConverter - } - outputConverterMap[name] = c - return nil -} diff --git a/lib/instance.go b/lib/instance.go deleted file mode 100644 index 5631766..0000000 --- a/lib/instance.go +++ /dev/null @@ -1,65 +0,0 @@ -package lib - -import ( - "encoding/json" - "errors" - "os" -) - -type Instance struct { - config *config - input []InputConverter - output []OutputConverter -} - -func NewInstance() (*Instance, error) { - return &Instance{ - config: new(config), - input: make([]InputConverter, 0), - output: make([]OutputConverter, 0), - }, nil -} - -func (i *Instance) Init(configFile string) error { - content, err := os.ReadFile(configFile) - if err != nil { - return err - } - - if err := json.Unmarshal(content, &i.config); err != nil { - return err - } - - for _, input := range i.config.Input { - i.input = append(i.input, input.converter) - } - - for _, output := range i.config.Output { - i.output = append(i.output, output.converter) - } - - return nil -} - -func (i *Instance) Run() error { - if len(i.input) == 0 || len(i.output) == 0 { - return errors.New("input type and output type must be specified") - } - - var err error - container := NewContainer() - for _, ic := range i.input { - container, err = ic.Input(container) - if err != nil { - return err - } - } - - for _, oc := range i.output { - if err := oc.Output(container); err != nil { - return err - } - } - - return nil -} diff --git a/lib/lib.go b/lib/lib.go deleted file mode 100644 index d1d23ab..0000000 --- a/lib/lib.go +++ /dev/null @@ -1,458 +0,0 @@ -package lib - -import ( - "fmt" - "log" - "net" - "net/netip" - "strings" - "sync" - - "go4.org/netipx" -) - -const ( - ActionAdd Action = "add" - ActionRemove Action = "remove" - ActionOutput Action = "output" - - IPv4 IPType = "ipv4" - IPv6 IPType = "ipv6" -) - -var ActionsRegistry = map[Action]bool{ - ActionAdd: true, - ActionRemove: true, - ActionOutput: true, -} - -type Action string - -type IPType string - -type Typer interface { - GetType() string -} - -type Actioner interface { - GetAction() Action -} - -type Descriptioner interface { - GetDescription() string -} - -type InputConverter interface { - Typer - Actioner - Descriptioner - Input(Container) (Container, error) -} - -type OutputConverter interface { - Typer - Actioner - Descriptioner - Output(Container) error -} - -type Entry struct { - name string - mu *sync.Mutex - ipv4Builder *netipx.IPSetBuilder - ipv6Builder *netipx.IPSetBuilder -} - -func NewEntry(name string) *Entry { - return &Entry{ - name: strings.ToUpper(strings.TrimSpace(name)), - mu: new(sync.Mutex), - ipv4Builder: new(netipx.IPSetBuilder), - ipv6Builder: new(netipx.IPSetBuilder), - } -} - -func (e *Entry) GetName() string { - return e.name -} - -func (e *Entry) hasIPv4Builder() bool { - return e.ipv4Builder != nil -} - -func (e *Entry) hasIPv6Builder() bool { - return e.ipv6Builder != nil -} - -func (e *Entry) processPrefix(src any) (*netip.Prefix, IPType, error) { - switch src := src.(type) { - case net.IP: - ip, ok := netipx.FromStdIP(src) - if !ok { - return nil, "", ErrInvalidIP - } - switch { - case ip.Is4(): - prefix := netip.PrefixFrom(ip, 32) - return &prefix, IPv4, nil - case ip.Is6(): - prefix := netip.PrefixFrom(ip, 128) - return &prefix, IPv6, nil - default: - return nil, "", ErrInvalidIPLength - } - - case *net.IPNet: - prefix, ok := netipx.FromStdIPNet(src) - if !ok { - return nil, "", ErrInvalidIPNet - } - ip := prefix.Addr() - switch { - case ip.Is4(): - return &prefix, IPv4, nil - case ip.Is6(): - return &prefix, IPv6, nil - default: - return nil, "", ErrInvalidIPLength - } - - case netip.Addr: - switch { - case src.Is4(): - prefix := netip.PrefixFrom(src, 32) - return &prefix, IPv4, nil - case src.Is6(): - prefix := netip.PrefixFrom(src, 128) - return &prefix, IPv6, nil - default: - return nil, "", ErrInvalidIPLength - } - - case *netip.Addr: - switch { - case src.Is4(): - prefix := netip.PrefixFrom(*src, 32) - return &prefix, IPv4, nil - case src.Is6(): - prefix := netip.PrefixFrom(*src, 128) - return &prefix, IPv6, nil - default: - return nil, "", ErrInvalidIPLength - } - - case netip.Prefix: - ip := src.Addr() - switch { - case ip.Is4(): - return &src, IPv4, nil - case ip.Is6(): - return &src, IPv6, nil - default: - return nil, "", ErrInvalidIPLength - } - - case *netip.Prefix: - ip := src.Addr() - switch { - case ip.Is4(): - return src, IPv4, nil - case ip.Is6(): - return src, IPv6, nil - default: - return nil, "", ErrInvalidIPLength - } - - case string: - _, network, err := net.ParseCIDR(src) - switch err { - case nil: - prefix, ok := netipx.FromStdIPNet(network) - if !ok { - return nil, "", ErrInvalidIPNet - } - ip := prefix.Addr() - switch { - case ip.Is4(): - return &prefix, IPv4, nil - case ip.Is6(): - return &prefix, IPv6, nil - default: - return nil, "", ErrInvalidIPLength - } - - default: - ip, err := netip.ParseAddr(src) - if err != nil { - return nil, "", err - } - switch { - case ip.Is4(): - prefix := netip.PrefixFrom(ip, 32) - return &prefix, IPv4, nil - case ip.Is6(): - prefix := netip.PrefixFrom(ip, 128) - return &prefix, IPv6, nil - default: - return nil, "", ErrInvalidIPLength - } - } - } - - return nil, "", ErrInvalidPrefixType -} - -func (e *Entry) add(prefix *netip.Prefix, ipType IPType) error { - e.mu.Lock() - defer e.mu.Unlock() - - switch ipType { - case IPv4: - if !e.hasIPv4Builder() { - e.ipv4Builder = new(netipx.IPSetBuilder) - } - e.ipv4Builder.AddPrefix(*prefix) - case IPv6: - if !e.hasIPv6Builder() { - e.ipv6Builder = new(netipx.IPSetBuilder) - } - e.ipv6Builder.AddPrefix(*prefix) - default: - return ErrInvalidIPType - } - - return nil -} - -func (e *Entry) remove(prefix *netip.Prefix, ipType IPType) error { - e.mu.Lock() - defer e.mu.Unlock() - - switch ipType { - case IPv4: - if e.hasIPv4Builder() { - e.ipv4Builder.RemovePrefix(*prefix) - } - case IPv6: - if e.hasIPv6Builder() { - e.ipv6Builder.RemovePrefix(*prefix) - } - default: - return ErrInvalidIPType - } - - return nil -} - -func (e *Entry) AddPrefix(cidr any) error { - prefix, ipType, err := e.processPrefix(cidr) - if err != nil { - return err - } - if err := e.add(prefix, ipType); err != nil { - return err - } - return nil -} - -func (e *Entry) RemovePrefix(cidr string) error { - prefix, ipType, err := e.processPrefix(cidr) - if err != nil { - return err - } - if err := e.remove(prefix, ipType); err != nil { - return err - } - return nil -} - -func (e *Entry) MarshalText(opts ...IgnoreIPOption) ([]string, error) { - var ignoreIPType IPType - for _, opt := range opts { - if opt != nil { - ignoreIPType = opt() - } - } - disableIPv4, disableIPv6 := false, false - switch ignoreIPType { - case IPv4: - disableIPv4 = true - case IPv6: - disableIPv6 = true - } - - prefixSet := make([]string, 0, 1024) - - if !disableIPv4 && e.hasIPv4Builder() { - ipv4set, err := e.ipv4Builder.IPSet() - if err != nil { - return nil, err - } - prefixes := ipv4set.Prefixes() - for _, prefix := range prefixes { - prefixSet = append(prefixSet, prefix.String()) - } - } - - if !disableIPv6 && e.hasIPv6Builder() { - ipv6set, err := e.ipv6Builder.IPSet() - if err != nil { - return nil, err - } - prefixes := ipv6set.Prefixes() - for _, prefix := range prefixes { - prefixSet = append(prefixSet, prefix.String()) - } - } - - if len(prefixSet) > 0 { - return prefixSet, nil - } - - return nil, fmt.Errorf("entry %s has no prefix", e.GetName()) -} - -type IgnoreIPOption func() IPType - -func IgnoreIPv4() IPType { - return IPv4 -} - -func IgnoreIPv6() IPType { - return IPv6 -} - -type Container interface { - GetEntry(name string) (*Entry, bool) - Add(entry *Entry, opts ...IgnoreIPOption) error - Remove(name string, opts ...IgnoreIPOption) - Loop() <-chan *Entry -} - -type container struct { - entries *sync.Map // map[name]*Entry -} - -func NewContainer() Container { - return &container{ - entries: new(sync.Map), - } -} - -func (c *container) isValid() bool { - if c == nil || c.entries == nil { - return false - } - return true -} - -func (c *container) GetEntry(name string) (*Entry, bool) { - if !c.isValid() { - return nil, false - } - val, ok := c.entries.Load(strings.ToUpper(strings.TrimSpace(name))) - if !ok { - return nil, false - } - return val.(*Entry), true -} - -func (c *container) Loop() <-chan *Entry { - ch := make(chan *Entry, 300) - go func() { - c.entries.Range(func(key, value any) bool { - ch <- value.(*Entry) - return true - }) - close(ch) - }() - return ch -} - -func (c *container) Add(entry *Entry, opts ...IgnoreIPOption) error { - var ignoreIPType IPType - for _, opt := range opts { - if opt != nil { - ignoreIPType = opt() - } - } - - name := entry.GetName() - val, found := c.GetEntry(name) - switch found { - case true: - var ipv4set, ipv6set *netipx.IPSet - var err4, err6 error - if entry.hasIPv4Builder() { - ipv4set, err4 = entry.ipv4Builder.IPSet() - if err4 != nil { - return err4 - } - } - if entry.hasIPv6Builder() { - ipv6set, err6 = entry.ipv6Builder.IPSet() - if err6 != nil { - return err6 - } - } - switch ignoreIPType { - case IPv4: - if !val.hasIPv6Builder() { - val.ipv6Builder = new(netipx.IPSetBuilder) - } - val.ipv6Builder.AddSet(ipv6set) - case IPv6: - if !val.hasIPv4Builder() { - val.ipv4Builder = new(netipx.IPSetBuilder) - } - val.ipv4Builder.AddSet(ipv4set) - default: - if !val.hasIPv4Builder() { - val.ipv4Builder = new(netipx.IPSetBuilder) - } - if !val.hasIPv6Builder() { - val.ipv6Builder = new(netipx.IPSetBuilder) - } - val.ipv4Builder.AddSet(ipv4set) - val.ipv6Builder.AddSet(ipv6set) - } - c.entries.Store(name, val) - - case false: - switch ignoreIPType { - case IPv4: - entry.ipv4Builder = nil - case IPv6: - entry.ipv6Builder = nil - } - c.entries.Store(name, entry) - } - - return nil -} - -func (c *container) Remove(name string, opts ...IgnoreIPOption) { - val, found := c.GetEntry(name) - if !found { - log.Printf("failed to remove non-existent entry %s", name) - return - } - - var ignoreIPType IPType - for _, opt := range opts { - if opt != nil { - ignoreIPType = opt() - } - } - - switch ignoreIPType { - case IPv4: - val.ipv6Builder = nil - c.entries.Store(name, val) - case IPv6: - val.ipv4Builder = nil - c.entries.Store(name, val) - default: - c.entries.Delete(name) - } -} diff --git a/main.go b/main.go deleted file mode 100644 index 7d954f1..0000000 --- a/main.go +++ /dev/null @@ -1,150 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "log" - "os" - "path/filepath" - "sort" - "strings" - - "github.com/v2fly/geoip/lib" - router "github.com/v2fly/v2ray-core/v5/app/router/routercommon" - "google.golang.org/protobuf/proto" -) - -var ( - // GeoIP flags - list = flag.Bool("l", false, "List all available input and output formats") - configFile = flag.String("c", "config.json", "Path to the config file") - - // Geosite flags - dataPath = flag.String("datapath", "./data", "Path to your custom 'data' directory") - outputName = flag.String("outputname", "dlc.dat", "Name of the generated dat file") - outputDir = flag.String("outputdir", "./", "Directory to place all generated files") - exportLists = flag.String("exportlists", "", "Lists to be flattened and exported in plaintext format, separated by ',' comma") - - mode = flag.String("m", "geoip", "Specify the mode to run: 'geoip' or 'geosite'") -) - -func main() { - flag.Parse() - - switch *mode { - case "geoip": - runGeoIP() - case "geosite": - runGeosite() - default: - log.Fatal("Unknown mode. Use 'geoip' or 'geosite'.") - } -} - -// GeoIP generation logic -func runGeoIP() { - if *list { - lib.ListInputConverter() - lib.ListOutputConverter() - return - } - - instance, err := lib.NewInstance() - if err != nil { - log.Fatal(err) - } - - if err := instance.Init(*configFile); err != nil { - log.Fatal(err) - } - - if err := instance.Run(); err != nil { - log.Fatal(err) - } -} - -// Geosite generation logic -func runGeosite() { - ref := make(map[string][]string) - - // Load and process all ref files - err := filepath.Walk(*dataPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if !info.IsDir() { - ref[info.Name()] = append(ref[info.Name()], path) - } - return nil - }) - if err != nil { - fmt.Println("Failed: ", err) - os.Exit(1) - } - - // Create output directory if not exist - if _, err := os.Stat(*outputDir); os.IsNotExist(err) { - if mkErr := os.MkdirAll(*outputDir, 0755); mkErr != nil { - fmt.Println("Failed: ", mkErr) - os.Exit(1) - } - } - - protoList := new(router.GeoSiteList) - var existList []string - for refName, list := range ref { - pl, err := ParseList(list, ref) - if err != nil { - fmt.Println("Failed: ", err) - os.Exit(1) - } - site, err := pl.toProto() - if err != nil { - fmt.Println("Failed: ", err) - os.Exit(1) - } - protoList.Entry = append(protoList.Entry, site) - - // Flatten and export plaintext list - if *exportLists != "" { - if existList != nil { - exportPlainTextList(existList, refName, pl) - } else { - exportedListSlice := strings.Split(*exportLists, ",") - for _, exportedListName := range exportedListSlice { - fileName := filepath.Join(*dataPath, exportedListName) - _, err := os.Stat(fileName) - if err == nil || os.IsExist(err) { - existList = append(existList, exportedListName) - } else { - fmt.Printf("'%s' list does not exist in '%s' directory.\n", exportedListName, *dataPath) - } - } - if existList != nil { - exportPlainTextList(existList, refName, pl) - } - } - } - } - - // Sort protoList so the marshaled list is reproducible - sort.SliceStable(protoList.Entry, func(i, j int) bool { - return protoList.Entry[i].CountryCode < protoList.Entry[j].CountryCode - }) - - protoBytes, err := proto.Marshal(protoList) - if err != nil { - fmt.Println("Failed:", err) - os.Exit(1) - } - if err := os.WriteFile(filepath.Join(*outputDir, *outputName), protoBytes, 0644); err != nil { - fmt.Println("Failed: ", err) - os.Exit(1) - } else { - fmt.Println(*outputName, "has been generated successfully.") - } -} - -// Additional utility functions for Geosite -// You may include ParseList, toProto, exportPlainTextList here if they are not yet defined in other packages. diff --git a/plugin/maxmind/country_csv.go b/plugin/maxmind/country_csv.go deleted file mode 100644 index 7773491..0000000 --- a/plugin/maxmind/country_csv.go +++ /dev/null @@ -1,214 +0,0 @@ -package maxmind - -import ( - "encoding/csv" - "encoding/json" - "errors" - "os" - "path/filepath" - "strings" - - "github.com/v2fly/geoip/lib" -) - -const ( - typeCountryCSV = "maxmindGeoLite2CountryCSV" - descCountryCSV = "Convert MaxMind GeoLite2 country CSV data to other formats" -) - -var ( - defaultCCFile = filepath.Join("./", "geolite2", "GeoLite2-Country-Locations-en.csv") - defaultIPv4File = filepath.Join("./", "geolite2", "GeoLite2-Country-Blocks-IPv4.csv") - defaultIPv6File = filepath.Join("./", "geolite2", "GeoLite2-Country-Blocks-IPv6.csv") -) - -func init() { - lib.RegisterInputConfigCreator(typeCountryCSV, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { - return newGeoLite2CountryCSV(action, data) - }) - lib.RegisterInputConverter(typeCountryCSV, &geoLite2CountryCSV{ - Description: descCountryCSV, - }) -} - -func newGeoLite2CountryCSV(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { - var tmp struct { - CountryCodeFile string `json:"country"` - IPv4File string `json:"ipv4"` - IPv6File string `json:"ipv6"` - Want []string `json:"wantedList"` - OnlyIPType lib.IPType `json:"onlyIPType"` - } - - if len(data) > 0 { - if err := json.Unmarshal(data, &tmp); err != nil { - return nil, err - } - } - - if tmp.CountryCodeFile == "" { - tmp.CountryCodeFile = defaultCCFile - } - - if tmp.IPv4File == "" { - tmp.IPv4File = defaultIPv4File - } - - if tmp.IPv6File == "" { - tmp.IPv6File = defaultIPv6File - } - - return &geoLite2CountryCSV{ - Type: typeCountryCSV, - Action: action, - Description: descCountryCSV, - CountryCodeFile: tmp.CountryCodeFile, - IPv4File: tmp.IPv4File, - IPv6File: tmp.IPv6File, - Want: tmp.Want, - OnlyIPType: tmp.OnlyIPType, - }, nil -} - -type geoLite2CountryCSV struct { - Type string - Action lib.Action - Description string - CountryCodeFile string - IPv4File string - IPv6File string - Want []string - OnlyIPType lib.IPType -} - -func (g *geoLite2CountryCSV) GetType() string { - return g.Type -} - -func (g *geoLite2CountryCSV) GetAction() lib.Action { - return g.Action -} - -func (g *geoLite2CountryCSV) GetDescription() string { - return g.Description -} - -func (g *geoLite2CountryCSV) Input(container lib.Container) (lib.Container, error) { - ccMap, err := g.getCountryCode() - if err != nil { - return nil, err - } - - entries := make(map[string]*lib.Entry) - - if g.IPv4File != "" { - if err := g.process(g.IPv4File, ccMap, entries); err != nil { - return nil, err - } - } - - if g.IPv6File != "" { - if err := g.process(g.IPv6File, ccMap, entries); err != nil { - return nil, err - } - } - - var ignoreIPType lib.IgnoreIPOption - switch g.OnlyIPType { - case lib.IPv4: - ignoreIPType = lib.IgnoreIPv6 - case lib.IPv6: - ignoreIPType = lib.IgnoreIPv4 - } - - for name, entry := range entries { - switch g.Action { - case lib.ActionAdd: - if err := container.Add(entry, ignoreIPType); err != nil { - return nil, err - } - case lib.ActionRemove: - container.Remove(name, ignoreIPType) - default: - return nil, lib.ErrUnknownAction - } - } - - return container, nil -} - -func (g *geoLite2CountryCSV) getCountryCode() (map[string]string, error) { - ccReader, err := os.Open(g.CountryCodeFile) - if err != nil { - return nil, err - } - defer ccReader.Close() - - reader := csv.NewReader(ccReader) - lines, err := reader.ReadAll() - if err != nil { - return nil, err - } - - ccMap := make(map[string]string) - for _, line := range lines[1:] { - id := strings.TrimSpace(line[0]) - countryCode := strings.TrimSpace(line[4]) - if id == "" || countryCode == "" { - continue - } - ccMap[id] = strings.ToUpper(countryCode) - } - return ccMap, nil -} - -func (g *geoLite2CountryCSV) process(file string, ccMap map[string]string, entries map[string]*lib.Entry) error { - if len(ccMap) == 0 { - return errors.New("country code list must be specified") - } - if entries == nil { - entries = make(map[string]*lib.Entry) - } - - fReader, err := os.Open(file) - if err != nil { - return err - } - defer fReader.Close() - - reader := csv.NewReader(fReader) - lines, err := reader.ReadAll() - if err != nil { - return err - } - - // Filter want list - wantList := make(map[string]bool) - for _, want := range g.Want { - if want = strings.ToUpper(strings.TrimSpace(want)); want != "" { - wantList[want] = true - } - } - - for _, line := range lines[1:] { - ccID := strings.TrimSpace(line[1]) - if countryCode, found := ccMap[ccID]; found { - if len(wantList) > 0 { - if _, found := wantList[countryCode]; !found { - continue - } - } - cidrStr := strings.ToLower(strings.TrimSpace(line[0])) - entry, found := entries[countryCode] - if !found { - entry = lib.NewEntry(countryCode) - } - if err := entry.AddPrefix(cidrStr); err != nil { - return err - } - entries[countryCode] = entry - } - } - - return nil -} diff --git a/plugin/maxmind/mmdb_in.go b/plugin/maxmind/mmdb_in.go deleted file mode 100644 index 9261f18..0000000 --- a/plugin/maxmind/mmdb_in.go +++ /dev/null @@ -1,227 +0,0 @@ -package maxmind - -import ( - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - - "github.com/oschwald/maxminddb-golang" - "github.com/v2fly/geoip/lib" -) - -const ( - typeMaxmindMMDBIn = "maxmindMMDB" - descMaxmindMMDBIn = "Convert MaxMind mmdb database to other formats" -) - -var ( - defaultMMDBFile = filepath.Join("./", "geolite2", "GeoLite2-Country.mmdb") - tempMMDBPath = filepath.Join("./", "tmp") - tempMMDBFile = filepath.Join(tempMMDBPath, "input.mmdb") -) - -func init() { - lib.RegisterInputConfigCreator(typeMaxmindMMDBIn, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { - return newMaxmindMMDBIn(action, data) - }) - lib.RegisterInputConverter(typeMaxmindMMDBIn, &maxmindMMDBIn{ - Description: descMaxmindMMDBIn, - }) -} - -func newMaxmindMMDBIn(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { - var tmp struct { - URI string `json:"uri"` - Want []string `json:"wantedList"` - OnlyIPType lib.IPType `json:"onlyIPType"` - } - - if len(data) > 0 { - if err := json.Unmarshal(data, &tmp); err != nil { - return nil, err - } - } - - if tmp.URI == "" { - tmp.URI = defaultMMDBFile - } - - return &maxmindMMDBIn{ - Type: typeMaxmindMMDBIn, - Action: action, - Description: descMaxmindMMDBIn, - URI: tmp.URI, - Want: tmp.Want, - OnlyIPType: tmp.OnlyIPType, - }, nil -} - -type maxmindMMDBIn struct { - Type string - Action lib.Action - Description string - URI string - Want []string - OnlyIPType lib.IPType -} - -func (g *maxmindMMDBIn) GetType() string { - return g.Type -} - -func (g *maxmindMMDBIn) GetAction() lib.Action { - return g.Action -} - -func (g *maxmindMMDBIn) GetDescription() string { - return g.Description -} - -func (g *maxmindMMDBIn) Input(container lib.Container) (lib.Container, error) { - var fd io.ReadCloser - var err error - switch { - case strings.HasPrefix(g.URI, "http://"), strings.HasPrefix(g.URI, "https://"): - fd, err = g.downloadFile(g.URI) - default: - fd, err = os.Open(g.URI) - } - - if err != nil { - return nil, err - } - - err = g.moveFile(fd) - if err != nil { - return nil, err - } - - entries := make(map[string]*lib.Entry) - err = g.generateEntries(entries) - if err != nil { - return nil, err - } - - if len(entries) == 0 { - return nil, fmt.Errorf("❌ [type %s | action %s] no entry is newly generated", typeMaxmindMMDBIn, g.Action) - } - - var ignoreIPType lib.IgnoreIPOption - switch g.OnlyIPType { - case lib.IPv4: - ignoreIPType = lib.IgnoreIPv6 - case lib.IPv6: - ignoreIPType = lib.IgnoreIPv4 - } - - // Filter want list - wantList := make(map[string]bool) - for _, want := range g.Want { - if want = strings.ToUpper(strings.TrimSpace(want)); want != "" { - wantList[want] = true - } - } - - for _, entry := range entries { - name := entry.GetName() - if len(wantList) > 0 && !wantList[name] { - continue - } - - switch g.Action { - case lib.ActionAdd: - if err := container.Add(entry, ignoreIPType); err != nil { - return nil, err - } - case lib.ActionRemove: - container.Remove(name, ignoreIPType) - } - } - - return container, nil -} - -func (g *maxmindMMDBIn) downloadFile(url string) (io.ReadCloser, error) { - resp, err := http.Get(url) - if err != nil { - return nil, err - } - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("failed to get remote file %s, http status code %d", url, resp.StatusCode) - } - - return resp.Body, nil -} - -func (g *maxmindMMDBIn) moveFile(src io.ReadCloser) error { - defer src.Close() - - err := os.MkdirAll(tempMMDBPath, 0755) - if err != nil { - return err - } - - out, err := os.Create(tempMMDBFile) - if err != nil { - return err - } - defer out.Close() - - _, err = io.Copy(out, src) - - return err -} - -func (g *maxmindMMDBIn) generateEntries(entries map[string]*lib.Entry) error { - db, err := maxminddb.Open(tempMMDBFile) - if err != nil { - return err - } - defer db.Close() - - record := struct { - Country struct { - IsoCode string `maxminddb:"iso_code"` - } `maxminddb:"country"` - }{} - - networks := db.Networks(maxminddb.SkipAliasedNetworks) - for networks.Next() { - subnet, err := networks.Network(&record) - if err != nil { - continue - } - - var entry *lib.Entry - name := strings.ToUpper(record.Country.IsoCode) - if theEntry, found := entries[name]; found { - entry = theEntry - } else { - entry = lib.NewEntry(name) - } - - switch g.Action { - case lib.ActionAdd: - if err := entry.AddPrefix(subnet); err != nil { - return err - } - case lib.ActionRemove: - if err := entry.RemovePrefix(subnet.String()); err != nil { - return err - } - } - - entries[name] = entry - } - - if networks.Err() != nil { - return networks.Err() - } - - return nil -} diff --git a/plugin/plaintext/text_in.go b/plugin/plaintext/text_in.go deleted file mode 100644 index f3fd7ed..0000000 --- a/plugin/plaintext/text_in.go +++ /dev/null @@ -1,227 +0,0 @@ -package plaintext - -import ( - "bufio" - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "regexp" - "strings" - - "github.com/v2fly/geoip/lib" -) - -const ( - typeTextIn = "text" - descTextIn = "Convert plaintext IP and CIDR to other formats" -) - -func init() { - lib.RegisterInputConfigCreator(typeTextIn, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { - return newTextIn(action, data) - }) - lib.RegisterInputConverter(typeTextIn, &textIn{ - Description: descTextIn, - }) -} - -func newTextIn(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { - var tmp struct { - Name string `json:"name"` - URI string `json:"uri"` - InputDir string `json:"inputDir"` - OnlyIPType lib.IPType `json:"onlyIPType"` - } - - if len(data) > 0 { - if err := json.Unmarshal(data, &tmp); err != nil { - return nil, err - } - } - - if tmp.Name == "" && tmp.URI == "" && tmp.InputDir == "" { - return nil, fmt.Errorf("type %s | action %s missing inputdir or name or uri", typeTextIn, action) - } - - if (tmp.Name != "" && tmp.URI == "") || (tmp.Name == "" && tmp.URI != "") { - return nil, fmt.Errorf("type %s | action %s name & uri must be specified together", typeTextIn, action) - } - - return &textIn{ - Type: typeTextIn, - Action: action, - Description: descTextIn, - Name: tmp.Name, - URI: tmp.URI, - InputDir: tmp.InputDir, - OnlyIPType: tmp.OnlyIPType, - }, nil -} - -type textIn struct { - Type string - Action lib.Action - Description string - Name string - URI string - InputDir string - OnlyIPType lib.IPType -} - -func (t *textIn) GetType() string { - return t.Type -} - -func (t *textIn) GetAction() lib.Action { - return t.Action -} - -func (t *textIn) GetDescription() string { - return t.Description -} - -func (t *textIn) Input(container lib.Container) (lib.Container, error) { - entries := make(map[string]*lib.Entry) - var err error - - switch { - case t.InputDir != "": - err = t.walkDir(t.InputDir, entries) - case t.Name != "" && t.URI != "": - switch { - case strings.HasPrefix(t.URI, "http://"), strings.HasPrefix(t.URI, "https://"): - err = t.walkRemoteFile(t.URI, t.Name, entries) - default: - err = t.walkLocalFile(t.URI, t.Name, entries) - } - default: - return nil, fmt.Errorf("config missing argument inputDir or name or uri") - } - - if err != nil { - return nil, err - } - - var ignoreIPType lib.IgnoreIPOption - switch t.OnlyIPType { - case lib.IPv4: - ignoreIPType = lib.IgnoreIPv6 - case lib.IPv6: - ignoreIPType = lib.IgnoreIPv4 - } - - if len(entries) == 0 { - return nil, fmt.Errorf("type %s | action %s no entry are generated", t.Type, t.Action) - } - - for _, entry := range entries { - switch t.Action { - case lib.ActionAdd: - if err := container.Add(entry, ignoreIPType); err != nil { - return nil, err - } - case lib.ActionRemove: - container.Remove(entry.GetName(), ignoreIPType) - } - } - - return container, nil -} - -func (t *textIn) walkDir(dir string, entries map[string]*lib.Entry) error { - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - - if err := t.walkLocalFile(path, "", entries); err != nil { - return err - } - - return nil - }) - - return err -} - -func (t *textIn) walkLocalFile(path, name string, entries map[string]*lib.Entry) error { - name = strings.TrimSpace(name) - var filename string - if name != "" { - filename = name - } else { - filename = filepath.Base(path) - } - - // check filename - if !regexp.MustCompile(`^[a-zA-Z0-9_.\-]+$`).MatchString(filename) { - return fmt.Errorf("filename %s cannot be entry name, please remove special characters in it", filename) - } - dotIndex := strings.LastIndex(filename, ".") - if dotIndex > 0 { - filename = filename[:dotIndex] - } - - if _, found := entries[filename]; found { - return fmt.Errorf("found duplicated file %s", filename) - } - - entry := lib.NewEntry(filename) - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - if err := t.scanFile(file, entry); err != nil { - return err - } - - entries[filename] = entry - - return nil -} - -func (t *textIn) walkRemoteFile(url, name string, entries map[string]*lib.Entry) error { - resp, err := http.Get(url) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return fmt.Errorf("failed to get remote file %s, http status code %d", url, resp.StatusCode) - } - - entry := lib.NewEntry(name) - if err := t.scanFile(resp.Body, entry); err != nil { - return err - } - - entries[name] = entry - return nil -} - -func (t *textIn) scanFile(reader io.Reader, entry *lib.Entry) error { - scanner := bufio.NewScanner(reader) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" { - continue - } - if err := entry.AddPrefix(line); err != nil { - return err - } - } - - if err := scanner.Err(); err != nil { - return err - } - - return nil -} diff --git a/plugin/plaintext/text_out.go b/plugin/plaintext/text_out.go deleted file mode 100644 index 68f1d92..0000000 --- a/plugin/plaintext/text_out.go +++ /dev/null @@ -1,166 +0,0 @@ -package plaintext - -import ( - "bytes" - "encoding/json" - "log" - "os" - "path/filepath" - "strings" - - "github.com/v2fly/geoip/lib" -) - -const ( - typeTextOut = "text" - descTextOut = "Convert data to plaintext CIDR format" -) - -var ( - defaultOutputDir = filepath.Join("./", "output", "text") -) - -func init() { - lib.RegisterOutputConfigCreator(typeTextOut, func(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) { - return newTextOut(action, data) - }) - lib.RegisterOutputConverter(typeTextOut, &textOut{ - Description: descTextOut, - }) -} - -func newTextOut(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) { - var tmp struct { - OutputDir string `json:"outputDir"` - Want []string `json:"wantedList"` - OnlyIPType lib.IPType `json:"onlyIPType"` - } - - if len(data) > 0 { - if err := json.Unmarshal(data, &tmp); err != nil { - return nil, err - } - } - - if tmp.OutputDir == "" { - tmp.OutputDir = defaultOutputDir - } - - return &textOut{ - Type: typeTextOut, - Action: action, - Description: descTextOut, - OutputDir: tmp.OutputDir, - Want: tmp.Want, - OnlyIPType: tmp.OnlyIPType, - }, nil -} - -type textOut struct { - Type string - Action lib.Action - Description string - OutputDir string - Want []string - OnlyIPType lib.IPType -} - -func (t *textOut) GetType() string { - return t.Type -} - -func (t *textOut) GetAction() lib.Action { - return t.Action -} - -func (t *textOut) GetDescription() string { - return t.Description -} - -func (t *textOut) Output(container lib.Container) error { - // Filter want list - wantList := make(map[string]bool) - for _, want := range t.Want { - if want = strings.ToUpper(strings.TrimSpace(want)); want != "" { - wantList[want] = true - } - } - - switch len(wantList) { - case 0: - for entry := range container.Loop() { - cidrList, err := t.marshalText(entry) - if err != nil { - return err - } - filename := strings.ToLower(entry.GetName()) + ".txt" - if err := t.writeFile(filename, cidrList); err != nil { - return err - } - } - - default: - for name := range wantList { - entry, found := container.GetEntry(name) - if !found { - log.Printf("❌ entry %s not found", name) - continue - } - cidrList, err := t.marshalText(entry) - if err != nil { - return err - } - filename := strings.ToLower(entry.GetName()) + ".txt" - if err := t.writeFile(filename, cidrList); err != nil { - return err - } - } - } - - return nil -} - -func (t *textOut) marshalText(entry *lib.Entry) ([]string, error) { - var entryCidr []string - var err error - switch t.OnlyIPType { - case lib.IPv4: - entryCidr, err = entry.MarshalText(lib.IgnoreIPv6) - if err != nil { - return nil, err - } - case lib.IPv6: - entryCidr, err = entry.MarshalText(lib.IgnoreIPv4) - if err != nil { - return nil, err - } - default: - entryCidr, err = entry.MarshalText() - if err != nil { - return nil, err - } - } - - return entryCidr, nil -} - -func (t *textOut) writeFile(filename string, cidrList []string) error { - var buf bytes.Buffer - for _, cidr := range cidrList { - buf.WriteString(cidr) - buf.WriteString("\n") - } - cidrBytes := buf.Bytes() - - if err := os.MkdirAll(t.OutputDir, 0755); err != nil { - return err - } - - if err := os.WriteFile(filepath.Join(t.OutputDir, filename), cidrBytes, 0644); err != nil { - return err - } - - log.Printf("✅ [%s] %s --> %s", t.Type, filename, t.OutputDir) - - return nil -} diff --git a/plugin/special/cutter.go b/plugin/special/cutter.go deleted file mode 100644 index 8a8fdba..0000000 --- a/plugin/special/cutter.go +++ /dev/null @@ -1,96 +0,0 @@ -package special - -import ( - "encoding/json" - "fmt" - "strings" - - "github.com/v2fly/geoip/lib" -) - -const ( - typeCutter = "cutter" - descCutter = "Remove data from previous steps" -) - -func init() { - lib.RegisterInputConfigCreator(typeCutter, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { - return newCutter(action, data) - }) - lib.RegisterInputConverter(typeCutter, &cutter{ - Description: descCutter, - }) -} - -func newCutter(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { - var tmp struct { - Want []string `json:"wantedList"` - OnlyIPType lib.IPType `json:"onlyIPType"` - } - - if len(data) > 0 { - if err := json.Unmarshal(data, &tmp); err != nil { - return nil, err - } - } - - if action != lib.ActionRemove { - return nil, fmt.Errorf("type %s only supports `remove` action", typeCutter) - } - - return &cutter{ - Type: typeCutter, - Action: action, - Description: descCutter, - Want: tmp.Want, - OnlyIPType: tmp.OnlyIPType, - }, nil -} - -type cutter struct { - Type string - Action lib.Action - Description string - Want []string - OnlyIPType lib.IPType -} - -func (c *cutter) GetType() string { - return c.Type -} - -func (c *cutter) GetAction() lib.Action { - return c.Action -} - -func (c *cutter) GetDescription() string { - return c.Description -} - -func (c *cutter) Input(container lib.Container) (lib.Container, error) { - var ignoreIPType lib.IgnoreIPOption - switch c.OnlyIPType { - case lib.IPv4: - ignoreIPType = lib.IgnoreIPv6 - case lib.IPv6: - ignoreIPType = lib.IgnoreIPv4 - } - - // Filter want list - wantList := make(map[string]bool) - for _, want := range c.Want { - if want = strings.ToUpper(strings.TrimSpace(want)); want != "" { - wantList[want] = true - } - } - - for entry := range container.Loop() { - name := entry.GetName() - if len(wantList) > 0 && !wantList[name] { - continue - } - container.Remove(name, ignoreIPType) - } - - return container, nil -} diff --git a/plugin/special/private.go b/plugin/special/private.go deleted file mode 100644 index e9dceca..0000000 --- a/plugin/special/private.go +++ /dev/null @@ -1,94 +0,0 @@ -package special - -import ( - "encoding/json" - - "github.com/v2fly/geoip/lib" -) - -const ( - entryNamePrivate = "private" - typePrivate = "private" - descPrivate = "Convert LAN and private network CIDR to other formats" -) - -var privateCIDRs = []string{ - "0.0.0.0/8", - "10.0.0.0/8", - "100.64.0.0/10", - "127.0.0.0/8", - "169.254.0.0/16", - "172.16.0.0/12", - "192.0.0.0/24", - "192.0.2.0/24", - "192.88.99.0/24", - "192.168.0.0/16", - "198.18.0.0/15", - "198.51.100.0/24", - "203.0.113.0/24", - "224.0.0.0/4", - "240.0.0.0/4", - "255.255.255.255/32", - "::/128", - "::1/128", - "fc00::/7", - "fe80::/10", - "ff00::/8", -} - -func init() { - lib.RegisterInputConfigCreator(typePrivate, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { - return newPrivate(action, data) - }) - lib.RegisterInputConverter(typePrivate, &private{ - Description: descPrivate, - }) -} - -func newPrivate(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { - return &private{ - Type: typePrivate, - Action: action, - Description: descPrivate, - }, nil -} - -type private struct { - Type string - Action lib.Action - Description string -} - -func (p *private) GetType() string { - return p.Type -} - -func (p *private) GetAction() lib.Action { - return p.Action -} - -func (p *private) GetDescription() string { - return p.Description -} - -func (p *private) Input(container lib.Container) (lib.Container, error) { - entry := lib.NewEntry(entryNamePrivate) - for _, cidr := range privateCIDRs { - if err := entry.AddPrefix(cidr); err != nil { - return nil, err - } - } - - switch p.Action { - case lib.ActionAdd: - if err := container.Add(entry); err != nil { - return nil, err - } - case lib.ActionRemove: - container.Remove(entryNamePrivate) - default: - return nil, lib.ErrUnknownAction - } - - return container, nil -} diff --git a/plugin/special/test.go b/plugin/special/test.go deleted file mode 100644 index 1a48893..0000000 --- a/plugin/special/test.go +++ /dev/null @@ -1,74 +0,0 @@ -package special - -import ( - "encoding/json" - - "github.com/v2fly/geoip/lib" -) - -const ( - entryNameTest = "test" - typeTest = "test" - descTest = "Convert specific CIDR to other formats (for test only)" -) - -var testCIDRs = []string{ - "127.0.0.0/8", -} - -func init() { - lib.RegisterInputConfigCreator(typeTest, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { - return newTest(action, data) - }) - lib.RegisterInputConverter(typeTest, &test{ - Description: descTest, - }) -} - -func newTest(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { - return &test{ - Type: typeTest, - Action: action, - Description: descTest, - }, nil -} - -type test struct { - Type string - Action lib.Action - Description string -} - -func (t *test) GetType() string { - return t.Type -} - -func (t *test) GetAction() lib.Action { - return t.Action -} - -func (t *test) GetDescription() string { - return t.Description -} - -func (t *test) Input(container lib.Container) (lib.Container, error) { - entry := lib.NewEntry(entryNameTest) - for _, cidr := range testCIDRs { - if err := entry.AddPrefix(cidr); err != nil { - return nil, err - } - } - - switch t.Action { - case lib.ActionAdd: - if err := container.Add(entry); err != nil { - return nil, err - } - case lib.ActionRemove: - container.Remove(entryNameTest) - default: - return nil, lib.ErrUnknownAction - } - - return container, nil -} diff --git a/plugin/v2ray/dat_in.go b/plugin/v2ray/dat_in.go deleted file mode 100644 index 03874a7..0000000 --- a/plugin/v2ray/dat_in.go +++ /dev/null @@ -1,204 +0,0 @@ -package v2ray - -import ( - "encoding/json" - "fmt" - "io" - "net" - "net/http" - "os" - "strings" - - "github.com/v2fly/geoip/lib" - router "github.com/v2fly/v2ray-core/v5/app/router/routercommon" - "google.golang.org/protobuf/proto" -) - -const ( - typeGeoIPdatIn = "v2rayGeoIPDat" - descGeoIPdatIn = "Convert V2Ray GeoIP dat to other formats" -) - -func init() { - lib.RegisterInputConfigCreator(typeGeoIPdatIn, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { - return newGeoIPDatIn(action, data) - }) - lib.RegisterInputConverter(typeGeoIPdatIn, &geoIPDatIn{ - Description: descGeoIPdatIn, - }) -} - -func newGeoIPDatIn(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { - var tmp struct { - URI string `json:"uri"` - Want []string `json:"wantedList"` - OnlyIPType lib.IPType `json:"onlyIPType"` - } - - if len(data) > 0 { - if err := json.Unmarshal(data, &tmp); err != nil { - return nil, err - } - } - - if tmp.URI == "" { - return nil, fmt.Errorf("[type %s | action %s] uri must be specified in config", typeGeoIPdatIn, action) - } - - return &geoIPDatIn{ - Type: typeGeoIPdatIn, - Action: action, - Description: descGeoIPdatIn, - URI: tmp.URI, - Want: tmp.Want, - OnlyIPType: tmp.OnlyIPType, - }, nil -} - -type geoIPDatIn struct { - Type string - Action lib.Action - Description string - URI string - Want []string - OnlyIPType lib.IPType -} - -func (g *geoIPDatIn) GetType() string { - return g.Type -} - -func (g *geoIPDatIn) GetAction() lib.Action { - return g.Action -} - -func (g *geoIPDatIn) GetDescription() string { - return g.Description -} - -func (g *geoIPDatIn) Input(container lib.Container) (lib.Container, error) { - entries := make(map[string]*lib.Entry) - var err error - - switch { - case strings.HasPrefix(g.URI, "http://"), strings.HasPrefix(g.URI, "https://"): - err = g.walkRemoteFile(g.URI, entries) - default: - err = g.walkLocalFile(g.URI, entries) - } - - if err != nil { - return nil, err - } - - if len(entries) == 0 { - return nil, fmt.Errorf("❌ [type %s | action %s] no entry is newly generated", typeGeoIPdatIn, g.Action) - } - - var ignoreIPType lib.IgnoreIPOption - switch g.OnlyIPType { - case lib.IPv4: - ignoreIPType = lib.IgnoreIPv6 - case lib.IPv6: - ignoreIPType = lib.IgnoreIPv4 - } - - // Filter want list - wantList := make(map[string]bool) - for _, want := range g.Want { - if want = strings.ToUpper(strings.TrimSpace(want)); want != "" { - wantList[want] = true - } - } - - for _, entry := range entries { - name := entry.GetName() - if len(wantList) > 0 && !wantList[name] { - continue - } - - switch g.Action { - case lib.ActionAdd: - if err := container.Add(entry, ignoreIPType); err != nil { - return nil, err - } - case lib.ActionRemove: - container.Remove(name, ignoreIPType) - } - } - - return container, nil -} - -func (g *geoIPDatIn) walkLocalFile(path string, entries map[string]*lib.Entry) error { - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - - if err := g.generateEntries(file, entries); err != nil { - return err - } - - return nil -} - -func (g *geoIPDatIn) walkRemoteFile(url string, entries map[string]*lib.Entry) error { - resp, err := http.Get(url) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return fmt.Errorf("failed to get remote file %s, http status code %d", url, resp.StatusCode) - } - - if err := g.generateEntries(resp.Body, entries); err != nil { - return err - } - - return nil -} - -func (g *geoIPDatIn) generateEntries(reader io.Reader, entries map[string]*lib.Entry) error { - geoipBytes, err := io.ReadAll(reader) - if err != nil { - return err - } - - var geoipList router.GeoIPList - if err := proto.Unmarshal(geoipBytes, &geoipList); err != nil { - return err - } - - for _, geoip := range geoipList.Entry { - var entry *lib.Entry - name := geoip.CountryCode - if theEntry, found := entries[name]; found { - fmt.Printf("⚠️ [type %s | action %s] found duplicated entry: %s. Process anyway\n", typeGeoIPdatIn, g.Action, name) - entry = theEntry - } else { - entry = lib.NewEntry(name) - } - - for _, v2rayCIDR := range geoip.Cidr { - ipStr := net.IP(v2rayCIDR.GetIp()).String() + "/" + fmt.Sprint(v2rayCIDR.GetPrefix()) - switch g.Action { - case lib.ActionAdd: - if err := entry.AddPrefix(ipStr); err != nil { - return err - } - case lib.ActionRemove: - if err := entry.RemovePrefix(ipStr); err != nil { - return err - } - } - } - - entries[name] = entry - } - - return nil -} diff --git a/plugin/v2ray/dat_out.go b/plugin/v2ray/dat_out.go deleted file mode 100644 index 3c4ba23..0000000 --- a/plugin/v2ray/dat_out.go +++ /dev/null @@ -1,227 +0,0 @@ -package v2ray - -import ( - "encoding/json" - "fmt" - "log" - "os" - "path/filepath" - "sort" - "strings" - - "github.com/v2fly/geoip/lib" - router "github.com/v2fly/v2ray-core/v5/app/router/routercommon" - "github.com/v2fly/v2ray-core/v5/infra/conf/rule" - "google.golang.org/protobuf/proto" -) - -const ( - typeGeoIPdatOut = "v2rayGeoIPDat" - descGeoIPdatOut = "Convert data to V2Ray GeoIP dat format" -) - -var ( - defaultOutputName = "geoip.dat" - defaultOutputDir = filepath.Join("./", "output", "dat") -) - -func init() { - lib.RegisterOutputConfigCreator(typeGeoIPdatOut, func(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) { - return newGeoIPDat(action, data) - }) - lib.RegisterOutputConverter(typeGeoIPdatOut, &geoIPDatOut{ - Description: descGeoIPdatOut, - }) -} - -func newGeoIPDat(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) { - var tmp struct { - OutputName string `json:"outputName"` - OutputDir string `json:"outputDir"` - Want []string `json:"wantedList"` - OneFilePerList bool `json:"oneFilePerList"` - OnlyIPType lib.IPType `json:"onlyIPType"` - } - - if len(data) > 0 { - if err := json.Unmarshal(data, &tmp); err != nil { - return nil, err - } - } - - if tmp.OutputName == "" { - tmp.OutputName = defaultOutputName - } - - if tmp.OutputDir == "" { - tmp.OutputDir = defaultOutputDir - } - - return &geoIPDatOut{ - Type: typeGeoIPdatOut, - Action: action, - Description: descGeoIPdatOut, - OutputName: tmp.OutputName, - OutputDir: tmp.OutputDir, - Want: tmp.Want, - OneFilePerList: tmp.OneFilePerList, - OnlyIPType: tmp.OnlyIPType, - }, nil -} - -type geoIPDatOut struct { - Type string - Action lib.Action - Description string - OutputName string - OutputDir string - Want []string - OneFilePerList bool - OnlyIPType lib.IPType -} - -func (g *geoIPDatOut) GetType() string { - return g.Type -} - -func (g *geoIPDatOut) GetAction() lib.Action { - return g.Action -} - -func (g *geoIPDatOut) GetDescription() string { - return g.Description -} - -func (g *geoIPDatOut) Output(container lib.Container) error { - // Filter want list - wantList := make(map[string]bool) - for _, want := range g.Want { - if want = strings.ToUpper(strings.TrimSpace(want)); want != "" { - wantList[want] = true - } - } - - geoIPList := new(router.GeoIPList) - geoIPList.Entry = make([]*router.GeoIP, 0, 300) - updated := false - switch len(wantList) { - case 0: - for entry := range container.Loop() { - geoIP, err := g.generateGeoIP(entry) - if err != nil { - return err - } - geoIPList.Entry = append(geoIPList.Entry, geoIP) - updated = true - - if g.OneFilePerList { - geoIPBytes, err := proto.Marshal(geoIPList) - if err != nil { - return err - } - filename := strings.ToLower(entry.GetName()) + ".dat" - if err := g.writeFile(filename, geoIPBytes); err != nil { - return err - } - geoIPList.Entry = nil - } - } - - default: - for name := range wantList { - entry, found := container.GetEntry(name) - if !found { - log.Printf("❌ entry %s not found", name) - continue - } - geoIP, err := g.generateGeoIP(entry) - if err != nil { - return err - } - geoIPList.Entry = append(geoIPList.Entry, geoIP) - updated = true - - if g.OneFilePerList { - geoIPBytes, err := proto.Marshal(geoIPList) - if err != nil { - return err - } - filename := strings.ToLower(entry.GetName()) + ".dat" - if err := g.writeFile(filename, geoIPBytes); err != nil { - return err - } - geoIPList.Entry = nil - } - } - } - - // Sort to make reproducible builds - g.sort(geoIPList) - - if !g.OneFilePerList && updated { - geoIPBytes, err := proto.Marshal(geoIPList) - if err != nil { - return err - } - if err := g.writeFile(g.OutputName, geoIPBytes); err != nil { - return err - } - } - - return nil -} - -func (g *geoIPDatOut) generateGeoIP(entry *lib.Entry) (*router.GeoIP, error) { - var entryCidr []string - var err error - switch g.OnlyIPType { - case lib.IPv4: - entryCidr, err = entry.MarshalText(lib.IgnoreIPv6) - case lib.IPv6: - entryCidr, err = entry.MarshalText(lib.IgnoreIPv4) - default: - entryCidr, err = entry.MarshalText() - } - if err != nil { - return nil, err - } - - v2rayCIDR := make([]*router.CIDR, 0, 1024) - for _, cidrStr := range entryCidr { - cidr, err := rule.ParseIP(cidrStr) - if err != nil { - return nil, err - } - v2rayCIDR = append(v2rayCIDR, cidr) - } - - if len(v2rayCIDR) > 0 { - return &router.GeoIP{ - CountryCode: entry.GetName(), - Cidr: v2rayCIDR, - }, nil - } - - return nil, fmt.Errorf("entry %s has no CIDR", entry.GetName()) -} - -// Sort by country code to make reproducible builds -func (g *geoIPDatOut) sort(list *router.GeoIPList) { - sort.SliceStable(list.Entry, func(i, j int) bool { - return list.Entry[i].CountryCode < list.Entry[j].CountryCode - }) -} - -func (g *geoIPDatOut) writeFile(filename string, geoIPBytes []byte) error { - if err := os.MkdirAll(g.OutputDir, 0755); err != nil { - return err - } - - if err := os.WriteFile(filepath.Join(g.OutputDir, filename), geoIPBytes, 0644); err != nil { - return err - } - - log.Printf("✅ [%s] %s --> %s", g.Type, filename, g.OutputDir) - - return nil -}