diff --git a/LICENSE b/LICENSE index a612ad981..74c8b0933 100644 --- a/LICENSE +++ b/LICENSE @@ -1,373 +1,21 @@ -Mozilla Public License Version 2.0 -================================== - -1. Definitions --------------- - -1.1. "Contributor" - means each individual or legal entity that creates, contributes to - the creation of, or owns Covered Software. - -1.2. "Contributor Version" - means the combination of the Contributions of others (if any) used - by a Contributor and that particular Contributor's Contribution. - -1.3. "Contribution" - means Covered Software of a particular Contributor. - -1.4. "Covered Software" - means Source Code Form to which the initial Contributor has attached - the notice in Exhibit A, the Executable Form of such Source Code - Form, and Modifications of such Source Code Form, in each case - including portions thereof. - -1.5. "Incompatible With Secondary Licenses" - means - - (a) that the initial Contributor has attached the notice described - in Exhibit B to the Covered Software; or - - (b) that the Covered Software was made available under the terms of - version 1.1 or earlier of the License, but not also under the - terms of a Secondary License. - -1.6. "Executable Form" - means any form of the work other than Source Code Form. - -1.7. "Larger Work" - means a work that combines Covered Software with other material, in - a separate file or files, that is not Covered Software. - -1.8. "License" - means this document. - -1.9. "Licensable" - means having the right to grant, to the maximum extent possible, - whether at the time of the initial grant or subsequently, any and - all of the rights conveyed by this License. - -1.10. "Modifications" - means any of the following: - - (a) any file in Source Code Form that results from an addition to, - deletion from, or modification of the contents of Covered - Software; or - - (b) any new file in Source Code Form that contains any Covered - Software. - -1.11. "Patent Claims" of a Contributor - means any patent claim(s), including without limitation, method, - process, and apparatus claims, in any patent Licensable by such - Contributor that would be infringed, but for the grant of the - License, by the making, using, selling, offering for sale, having - made, import, or transfer of either its Contributions or its - Contributor Version. - -1.12. "Secondary License" - means either the GNU General Public License, Version 2.0, the GNU - Lesser General Public License, Version 2.1, the GNU Affero General - Public License, Version 3.0, or any later versions of those - licenses. - -1.13. "Source Code Form" - means the form of the work preferred for making modifications. - -1.14. "You" (or "Your") - means an individual or a legal entity exercising rights under this - License. For legal entities, "You" includes any entity that - controls, is controlled by, or is under common control with You. For - purposes of this definition, "control" means (a) the power, direct - or indirect, to cause the direction or management of such entity, - whether by contract or otherwise, or (b) ownership of more than - fifty percent (50%) of the outstanding shares or beneficial - ownership of such entity. - -2. License Grants and Conditions --------------------------------- - -2.1. Grants - -Each Contributor hereby grants You a world-wide, royalty-free, -non-exclusive license: - -(a) under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or - as part of a Larger Work; and - -(b) under Patent Claims of such Contributor to make, use, sell, offer - for sale, have made, import, and otherwise transfer either its - Contributions or its Contributor Version. - -2.2. Effective Date - -The licenses granted in Section 2.1 with respect to any Contribution -become effective for each Contribution on the date the Contributor first -distributes such Contribution. - -2.3. Limitations on Grant Scope - -The licenses granted in this Section 2 are the only rights granted under -this License. No additional rights or licenses will be implied from the -distribution or licensing of Covered Software under this License. -Notwithstanding Section 2.1(b) above, no patent license is granted by a -Contributor: - -(a) for any code that a Contributor has removed from Covered Software; - or - -(b) for infringements caused by: (i) Your and any other third party's - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or - -(c) under Patent Claims infringed by Covered Software in the absence of - its Contributions. - -This License does not grant any rights in the trademarks, service marks, -or logos of any Contributor (except as may be necessary to comply with -the notice requirements in Section 3.4). - -2.4. Subsequent Licenses - -No Contributor makes additional grants as a result of Your choice to -distribute the Covered Software under a subsequent version of this -License (see Section 10.2) or under the terms of a Secondary License (if -permitted under the terms of Section 3.3). - -2.5. Representation - -Each Contributor represents that the Contributor believes its -Contributions are its original creation(s) or it has sufficient rights -to grant the rights to its Contributions conveyed by this License. - -2.6. Fair Use - -This License is not intended to limit any rights You have under -applicable copyright doctrines of fair use, fair dealing, or other -equivalents. - -2.7. Conditions - -Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted -in Section 2.1. - -3. Responsibilities -------------------- - -3.1. Distribution of Source Form - -All distribution of Covered Software in Source Code Form, including any -Modifications that You create or to which You contribute, must be under -the terms of this License. You must inform recipients that the Source -Code Form of the Covered Software is governed by the terms of this -License, and how they can obtain a copy of this License. You may not -attempt to alter or restrict the recipients' rights in the Source Code -Form. - -3.2. Distribution of Executable Form - -If You distribute Covered Software in Executable Form then: - -(a) such Covered Software must also be made available in Source Code - Form, as described in Section 3.1, and You must inform recipients of - the Executable Form how they can obtain a copy of such Source Code - Form by reasonable means in a timely manner, at a charge no more - than the cost of distribution to the recipient; and - -(b) You may distribute such Executable Form under the terms of this - License, or sublicense it under different terms, provided that the - license for the Executable Form does not attempt to limit or alter - the recipients' rights in the Source Code Form under this License. - -3.3. Distribution of a Larger Work - -You may create and distribute a Larger Work under terms of Your choice, -provided that You also comply with the requirements of this License for -the Covered Software. If the Larger Work is a combination of Covered -Software with a work governed by one or more Secondary Licenses, and the -Covered Software is not Incompatible With Secondary Licenses, this -License permits You to additionally distribute such Covered Software -under the terms of such Secondary License(s), so that the recipient of -the Larger Work may, at their option, further distribute the Covered -Software under the terms of either this License or such Secondary -License(s). - -3.4. Notices - -You may not remove or alter the substance of any license notices -(including copyright notices, patent notices, disclaimers of warranty, -or limitations of liability) contained within the Source Code Form of -the Covered Software, except that You may alter any license notices to -the extent required to remedy known factual inaccuracies. - -3.5. Application of Additional Terms - -You may choose to offer, and to charge a fee for, warranty, support, -indemnity or liability obligations to one or more recipients of Covered -Software. However, You may do so only on Your own behalf, and not on -behalf of any Contributor. You must make it absolutely clear that any -such warranty, support, indemnity, or liability obligation is offered by -You alone, and You hereby agree to indemnify every Contributor for any -liability incurred by such Contributor as a result of warranty, support, -indemnity or liability terms You offer. You may include additional -disclaimers of warranty and limitations of liability specific to any -jurisdiction. - -4. Inability to Comply Due to Statute or Regulation ---------------------------------------------------- - -If it is impossible for You to comply with any of the terms of this -License with respect to some or all of the Covered Software due to -statute, judicial order, or regulation then You must: (a) comply with -the terms of this License to the maximum extent possible; and (b) -describe the limitations and the code they affect. Such description must -be placed in a text file included with all distributions of the Covered -Software under this License. Except to the extent prohibited by statute -or regulation, such description must be sufficiently detailed for a -recipient of ordinary skill to be able to understand it. - -5. Termination --------------- - -5.1. The rights granted under this License will terminate automatically -if You fail to comply with any of its terms. However, if You become -compliant, then the rights granted under this License from a particular -Contributor are reinstated (a) provisionally, unless and until such -Contributor explicitly and finally terminates Your grants, and (b) on an -ongoing basis, if such Contributor fails to notify You of the -non-compliance by some reasonable means prior to 60 days after You have -come back into compliance. Moreover, Your grants from a particular -Contributor are reinstated on an ongoing basis if such Contributor -notifies You of the non-compliance by some reasonable means, this is the -first time You have received notice of non-compliance with this License -from such Contributor, and You become compliant prior to 30 days after -Your receipt of the notice. - -5.2. If You initiate litigation against any entity by asserting a patent -infringement claim (excluding declaratory judgment actions, -counter-claims, and cross-claims) alleging that a Contributor Version -directly or indirectly infringes any patent, then the rights granted to -You by any and all Contributors for the Covered Software under Section -2.1 of this License shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all -end user license agreements (excluding distributors and resellers) which -have been validly granted by You or Your distributors under this License -prior to termination shall survive termination. - -************************************************************************ -* * -* 6. Disclaimer of Warranty * -* ------------------------- * -* * -* Covered Software is provided under this License on an "as is" * -* basis, without warranty of any kind, either expressed, implied, or * -* statutory, including, without limitation, warranties that the * -* Covered Software is free of defects, merchantable, fit for a * -* particular purpose or non-infringing. The entire risk as to the * -* quality and performance of the Covered Software is with You. * -* Should any Covered Software prove defective in any respect, You * -* (not any Contributor) assume the cost of any necessary servicing, * -* repair, or correction. This disclaimer of warranty constitutes an * -* essential part of this License. No use of any Covered Software is * -* authorized under this License except under this disclaimer. * -* * -************************************************************************ - -************************************************************************ -* * -* 7. Limitation of Liability * -* -------------------------- * -* * -* Under no circumstances and under no legal theory, whether tort * -* (including negligence), contract, or otherwise, shall any * -* Contributor, or anyone who distributes Covered Software as * -* permitted above, be liable to You for any direct, indirect, * -* special, incidental, or consequential damages of any character * -* including, without limitation, damages for lost profits, loss of * -* goodwill, work stoppage, computer failure or malfunction, or any * -* and all other commercial damages or losses, even if such party * -* shall have been informed of the possibility of such damages. This * -* limitation of liability shall not apply to liability for death or * -* personal injury resulting from such party's negligence to the * -* extent applicable law prohibits such limitation. Some * -* jurisdictions do not allow the exclusion or limitation of * -* incidental or consequential damages, so this exclusion and * -* limitation may not apply to You. * -* * -************************************************************************ - -8. Litigation -------------- - -Any litigation relating to this License may be brought only in the -courts of a jurisdiction where the defendant maintains its principal -place of business and such litigation shall be governed by laws of that -jurisdiction, without reference to its conflict-of-law provisions. -Nothing in this Section shall prevent a party's ability to bring -cross-claims or counter-claims. - -9. Miscellaneous ----------------- - -This License represents the complete agreement concerning the subject -matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent -necessary to make it enforceable. Any law or regulation which provides -that the language of a contract shall be construed against the drafter -shall not be used to construe this License against a Contributor. - -10. Versions of the License ---------------------------- - -10.1. New Versions - -Mozilla Foundation is the license steward. Except as provided in Section -10.3, no one other than the license steward has the right to modify or -publish new versions of this License. Each version will be given a -distinguishing version number. - -10.2. Effect of New Versions - -You may distribute the Covered Software under the terms of the version -of the License under which You originally received the Covered Software, -or under the terms of any subsequent version published by the license -steward. - -10.3. Modified Versions - -If you create software not governed by this License, and you want to -create a new license for such software, you may create and use a -modified version of this License if you rename the license and remove -any references to the name of the license steward (except to note that -such modified license differs from this License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary -Licenses - -If You choose to distribute Source Code Form that is Incompatible With -Secondary Licenses under the terms of this version of the License, the -notice described in Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice -------------------------------------------- - - This Source Code Form is subject to the terms of the Mozilla Public - License, v. 2.0. If a copy of the MPL was not distributed with this - file, You can obtain one at http://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular -file, then You may include the notice in a location (such as a LICENSE -file in a relevant directory) where a recipient would be likely to look -for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - "Incompatible With Secondary Licenses" Notice ---------------------------------------------------------- - - This Source Code Form is "Incompatible With Secondary Licenses", as - defined by the Mozilla Public License, v. 2.0. +MIT License + +Copyright (c) 2017 john@johng.cn + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/doc/.keep b/doc/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/g/core/gmvc/controller.go b/g/core/gmvc/controller.go new file mode 100644 index 000000000..c5f6faf85 --- /dev/null +++ b/g/core/gmvc/controller.go @@ -0,0 +1,5 @@ +package gmvc + + + + diff --git a/g/core/gmvc/model.go b/g/core/gmvc/model.go new file mode 100644 index 000000000..f591af65e --- /dev/null +++ b/g/core/gmvc/model.go @@ -0,0 +1,3 @@ +package gmvc + + diff --git a/g/core/gmvc/view.go b/g/core/gmvc/view.go new file mode 100644 index 000000000..f591af65e --- /dev/null +++ b/g/core/gmvc/view.go @@ -0,0 +1,3 @@ +package gmvc + + diff --git a/g/core/types/gbtree/btree_mem.go b/g/core/types/gbtree/btree_mem.go new file mode 100644 index 000000000..70bf79bf6 --- /dev/null +++ b/g/core/types/gbtree/btree_mem.go @@ -0,0 +1,76 @@ +// Copyright 2014 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build ignore + +// This binary compares memory usage between btree and gollrb. +package gbtree + +import ( + "flag" + "fmt" + "math/rand" + "runtime" + "time" + + "github.com/google/btree" + "github.com/petar/GoLLRB/llrb" +) + +var ( + size = flag.Int("size", 1000000, "size of the tree to build") + degree = flag.Int("degree", 8, "degree of btree") + gollrb = flag.Bool("llrb", false, "use llrb instead of btree") +) + +func main() { + flag.Parse() + vals := rand.Perm(*size) + var t, v interface{} + v = vals + var stats runtime.MemStats + for i := 0; i < 10; i++ { + runtime.GC() + } + fmt.Println("-------- BEFORE ----------") + runtime.ReadMemStats(&stats) + fmt.Printf("%+v\n", stats) + start := time.Now() + if *gollrb { + tr := llrb.New() + for _, v := range vals { + tr.ReplaceOrInsert(llrb.Int(v)) + } + t = tr // keep it around + } else { + tr := btree.New(*degree) + for _, v := range vals { + tr.ReplaceOrInsert(btree.Int(v)) + } + t = tr // keep it around + } + fmt.Printf("%v inserts in %v\n", *size, time.Since(start)) + fmt.Println("-------- AFTER ----------") + runtime.ReadMemStats(&stats) + fmt.Printf("%+v\n", stats) + for i := 0; i < 10; i++ { + runtime.GC() + } + fmt.Println("-------- AFTER GC ----------") + runtime.ReadMemStats(&stats) + fmt.Printf("%+v\n", stats) + if t == v { + fmt.Println("to make sure vals and tree aren't GC'd") + } +} \ No newline at end of file diff --git a/g/core/types/gbtree/gbtree.go b/g/core/types/gbtree/gbtree.go new file mode 100644 index 000000000..e7da6dca5 --- /dev/null +++ b/g/core/types/gbtree/gbtree.go @@ -0,0 +1,776 @@ +// B+树，来源于：from https://github.com/google/btree +package gbtree + +import ( + "fmt" + "io" + "sort" + "strings" + "sync" +) + +// Item represents a single object in the tree. +type Item interface { + // Less tests whether the current item is less than the given argument. + // + // This must provide a strict weak ordering. + // If !a.Less(b) && !b.Less(a), we treat this to mean a == b (i.e. we can only + // hold one of either a or b in the tree). + Less(than Item) bool +} + +const ( + DefaultFreeListSize = 32 +) + +var ( + nilItems = make(items, 16) + nilChildren = make(children, 16) +) + +// FreeList represents a free list of btree nodes. By default each +// BTree has its own FreeList, but multiple BTrees can share the same +// FreeList. +// Two Btrees using the same freelist are safe for concurrent write access. +type FreeList struct { + mu sync.Mutex + freelist []*node +} + +// NewFreeList creates a new free list. +// size is the maximum size of the returned free list. +func NewFreeList(size int) *FreeList { + return &FreeList{freelist: make([]*node, 0, size)} +} + +func (f *FreeList) newNode() (n *node) { + f.mu.Lock() + index := len(f.freelist) - 1 + if index < 0 { + f.mu.Unlock() + return new(node) + } + n = f.freelist[index] + f.freelist[index] = nil + f.freelist = f.freelist[:index] + f.mu.Unlock() + return +} + +func (f *FreeList) freeNode(n *node) { + f.mu.Lock() + if len(f.freelist) < cap(f.freelist) { + f.freelist = append(f.freelist, n) + } + f.mu.Unlock() +} + +// ItemIterator allows callers of Ascend* to iterate in-order over portions of +// the tree. When this function returns false, iteration will stop and the +// associated Ascend* function will immediately return. +type ItemIterator func(i Item) bool + +// New creates a new B-Tree with the given degree. +// +// New(2), for example, will create a 2-3-4 tree (each node contains 1-3 items +// and 2-4 children). +func New(degree int) *BTree { + return NewWithFreeList(degree, NewFreeList(DefaultFreeListSize)) +} + +// NewWithFreeList creates a new B-Tree that uses the given node free list. +func NewWithFreeList(degree int, f *FreeList) *BTree { + if degree <= 1 { + panic("bad degree") + } + return &BTree{ + degree: degree, + cow: ©OnWriteContext{freelist: f}, + } +} + +// items stores items in a node. +type items []Item + +// insertAt inserts a value into the given index, pushing all subsequent values +// forward. +func (s *items) insertAt(index int, item Item) { + *s = append(*s, nil) + if index < len(*s) { + copy((*s)[index+1:], (*s)[index:]) + } + (*s)[index] = item +} + +// removeAt removes a value at a given index, pulling all subsequent values +// back. +func (s *items) removeAt(index int) Item { + item := (*s)[index] + copy((*s)[index:], (*s)[index+1:]) + (*s)[len(*s)-1] = nil + *s = (*s)[:len(*s)-1] + return item +} + +// pop removes and returns the last element in the list. +func (s *items) pop() (out Item) { + index := len(*s) - 1 + out = (*s)[index] + (*s)[index] = nil + *s = (*s)[:index] + return +} + +// truncate truncates this instance at index so that it contains only the +// first index items. index must be less than or equal to length. +func (s *items) truncate(index int) { + var toClear items + *s, toClear = (*s)[:index], (*s)[index:] + for len(toClear) > 0 { + toClear = toClear[copy(toClear, nilItems):] + } +} + +// find returns the index where the given item should be inserted into this +// list. 'found' is true if the item already exists in the list at the given +// index. +func (s items) find(item Item) (index int, found bool) { + i := sort.Search(len(s), func(i int) bool { + return item.Less(s[i]) + }) + if i > 0 && !s[i-1].Less(item) { + return i - 1, true + } + return i, false +} + +// children stores child nodes in a node. +type children []*node + +// insertAt inserts a value into the given index, pushing all subsequent values +// forward. +func (s *children) insertAt(index int, n *node) { + *s = append(*s, nil) + if index < len(*s) { + copy((*s)[index+1:], (*s)[index:]) + } + (*s)[index] = n +} + +// removeAt removes a value at a given index, pulling all subsequent values +// back. +func (s *children) removeAt(index int) *node { + n := (*s)[index] + copy((*s)[index:], (*s)[index+1:]) + (*s)[len(*s)-1] = nil + *s = (*s)[:len(*s)-1] + return n +} + +// pop removes and returns the last element in the list. +func (s *children) pop() (out *node) { + index := len(*s) - 1 + out = (*s)[index] + (*s)[index] = nil + *s = (*s)[:index] + return +} + +// truncate truncates this instance at index so that it contains only the +// first index children. index must be less than or equal to length. +func (s *children) truncate(index int) { + var toClear children + *s, toClear = (*s)[:index], (*s)[index:] + for len(toClear) > 0 { + toClear = toClear[copy(toClear, nilChildren):] + } +} + +// node is an internal node in a tree. +// +// It must at all times maintain the invariant that either +// * len(children) == 0, len(items) unconstrained +// * len(children) == len(items) + 1 +type node struct { + items items + children children + cow *copyOnWriteContext +} + +func (n *node) mutableFor(cow *copyOnWriteContext) *node { + if n.cow == cow { + return n + } + out := cow.newNode() + if cap(out.items) >= len(n.items) { + out.items = out.items[:len(n.items)] + } else { + out.items = make(items, len(n.items), cap(n.items)) + } + copy(out.items, n.items) + // Copy children + if cap(out.children) >= len(n.children) { + out.children = out.children[:len(n.children)] + } else { + out.children = make(children, len(n.children), cap(n.children)) + } + copy(out.children, n.children) + return out +} + +func (n *node) mutableChild(i int) *node { + c := n.children[i].mutableFor(n.cow) + n.children[i] = c + return c +} + +// split splits the given node at the given index. The current node shrinks, +// and this function returns the item that existed at that index and a new node +// containing all items/children after it. +func (n *node) split(i int) (Item, *node) { + item := n.items[i] + next := n.cow.newNode() + next.items = append(next.items, n.items[i+1:]...) + n.items.truncate(i) + if len(n.children) > 0 { + next.children = append(next.children, n.children[i+1:]...) + n.children.truncate(i + 1) + } + return item, next +} + +// maybeSplitChild checks if a child should be split, and if so splits it. +// Returns whether or not a split occurred. +func (n *node) maybeSplitChild(i, maxItems int) bool { + if len(n.children[i].items) < maxItems { + return false + } + first := n.mutableChild(i) + item, second := first.split(maxItems / 2) + n.items.insertAt(i, item) + n.children.insertAt(i+1, second) + return true +} + +// insert inserts an item into the subtree rooted at this node, making sure +// no nodes in the subtree exceed maxItems items. Should an equivalent item be +// be found/replaced by insert, it will be returned. +func (n *node) insert(item Item, maxItems int) Item { + i, found := n.items.find(item) + if found { + out := n.items[i] + n.items[i] = item + return out + } + if len(n.children) == 0 { + n.items.insertAt(i, item) + return nil + } + if n.maybeSplitChild(i, maxItems) { + inTree := n.items[i] + switch { + case item.Less(inTree): + // no change, we want first split node + case inTree.Less(item): + i++ // we want second split node + default: + out := n.items[i] + n.items[i] = item + return out + } + } + return n.mutableChild(i).insert(item, maxItems) +} + +// get finds the given key in the subtree and returns it. +func (n *node) get(key Item) Item { + i, found := n.items.find(key) + if found { + return n.items[i] + } else if len(n.children) > 0 { + return n.children[i].get(key) + } + return nil +} + +// min returns the first item in the subtree. +func min(n *node) Item { + if n == nil { + return nil + } + for len(n.children) > 0 { + n = n.children[0] + } + if len(n.items) == 0 { + return nil + } + return n.items[0] +} + +// max returns the last item in the subtree. +func max(n *node) Item { + if n == nil { + return nil + } + for len(n.children) > 0 { + n = n.children[len(n.children)-1] + } + if len(n.items) == 0 { + return nil + } + return n.items[len(n.items)-1] +} + +// toRemove details what item to remove in a node.remove call. +type toRemove int + +const ( + removeItem toRemove = iota // removes the given item + removeMin // removes smallest item in the subtree + removeMax // removes largest item in the subtree +) + +// remove removes an item from the subtree rooted at this node. +func (n *node) remove(item Item, minItems int, typ toRemove) Item { + var i int + var found bool + switch typ { + case removeMax: + if len(n.children) == 0 { + return n.items.pop() + } + i = len(n.items) + case removeMin: + if len(n.children) == 0 { + return n.items.removeAt(0) + } + i = 0 + case removeItem: + i, found = n.items.find(item) + if len(n.children) == 0 { + if found { + return n.items.removeAt(i) + } + return nil + } + default: + panic("invalid type") + } + // If we get to here, we have children. + if len(n.children[i].items) <= minItems { + return n.growChildAndRemove(i, item, minItems, typ) + } + child := n.mutableChild(i) + // Either we had enough items to begin with, or we've done some + // merging/stealing, because we've got enough now and we're ready to return + // stuff. + if found { + // The item exists at index 'i', and the child we've selected can give us a + // predecessor, since if we've gotten here it's got > minItems items in it. + out := n.items[i] + // We use our special-case 'remove' call with typ=maxItem to pull the + // predecessor of item i (the rightmost leaf of our immediate left child) + // and set it into where we pulled the item from. + n.items[i] = child.remove(nil, minItems, removeMax) + return out + } + // Final recursive call. Once we're here, we know that the item isn't in this + // node and that the child is big enough to remove from. + return child.remove(item, minItems, typ) +} + +// growChildAndRemove grows child 'i' to make sure it's possible to remove an +// item from it while keeping it at minItems, then calls remove to actually +// remove it. +// +// Most documentation says we have to do two sets of special casing: +// 1) item is in this node +// 2) item is in child +// In both cases, we need to handle the two subcases: +// A) node has enough values that it can spare one +// B) node doesn't have enough values +// For the latter, we have to check: +// a) left sibling has node to spare +// b) right sibling has node to spare +// c) we must merge +// To simplify our code here, we handle cases #1 and #2 the same: +// If a node doesn't have enough items, we make sure it does (using a,b,c). +// We then simply redo our remove call, and the second time (regardless of +// whether we're in case 1 or 2), we'll have enough items and can guarantee +// that we hit case A. +func (n *node) growChildAndRemove(i int, item Item, minItems int, typ toRemove) Item { + if i > 0 && len(n.children[i-1].items) > minItems { + // Steal from left child + child := n.mutableChild(i) + stealFrom := n.mutableChild(i - 1) + stolenItem := stealFrom.items.pop() + child.items.insertAt(0, n.items[i-1]) + n.items[i-1] = stolenItem + if len(stealFrom.children) > 0 { + child.children.insertAt(0, stealFrom.children.pop()) + } + } else if i < len(n.items) && len(n.children[i+1].items) > minItems { + // steal from right child + child := n.mutableChild(i) + stealFrom := n.mutableChild(i + 1) + stolenItem := stealFrom.items.removeAt(0) + child.items = append(child.items, n.items[i]) + n.items[i] = stolenItem + if len(stealFrom.children) > 0 { + child.children = append(child.children, stealFrom.children.removeAt(0)) + } + } else { + if i >= len(n.items) { + i-- + } + child := n.mutableChild(i) + // merge with right child + mergeItem := n.items.removeAt(i) + mergeChild := n.children.removeAt(i + 1) + child.items = append(child.items, mergeItem) + child.items = append(child.items, mergeChild.items...) + child.children = append(child.children, mergeChild.children...) + n.cow.freeNode(mergeChild) + } + return n.remove(item, minItems, typ) +} + +type direction int + +const ( + descend = direction(-1) + ascend = direction(+1) +) + +// iterate provides a simple method for iterating over elements in the tree. +// +// When ascending, the 'start' should be less than 'stop' and when descending, +// the 'start' should be greater than 'stop'. Setting 'includeStart' to true +// will force the iterator to include the first item when it equals 'start', +// thus creating a "greaterOrEqual" or "lessThanEqual" rather than just a +// "greaterThan" or "lessThan" queries. +func (n *node) iterate(dir direction, start, stop Item, includeStart bool, hit bool, iter ItemIterator) (bool, bool) { + var ok bool + switch dir { + case ascend: + for i := 0; i < len(n.items); i++ { + if start != nil && n.items[i].Less(start) { + continue + } + if len(n.children) > 0 { + if hit, ok = n.children[i].iterate(dir, start, stop, includeStart, hit, iter); !ok { + return hit, false + } + } + if !includeStart && !hit && start != nil && !start.Less(n.items[i]) { + hit = true + continue + } + hit = true + if stop != nil && !n.items[i].Less(stop) { + return hit, false + } + if !iter(n.items[i]) { + return hit, false + } + } + if len(n.children) > 0 { + if hit, ok = n.children[len(n.children)-1].iterate(dir, start, stop, includeStart, hit, iter); !ok { + return hit, false + } + } + + case descend: + for i := len(n.items) - 1; i >= 0; i-- { + if start != nil && !n.items[i].Less(start) { + if !includeStart || hit || start.Less(n.items[i]) { + continue + } + } + if len(n.children) > 0 { + if hit, ok = n.children[i+1].iterate(dir, start, stop, includeStart, hit, iter); !ok { + return hit, false + } + } + if stop != nil && !stop.Less(n.items[i]) { + return hit, false // continue + } + hit = true + if !iter(n.items[i]) { + return hit, false + } + } + if len(n.children) > 0 { + if hit, ok = n.children[0].iterate(dir, start, stop, includeStart, hit, iter); !ok { + return hit, false + } + } + } + return hit, true +} + +// Used for testing/debugging purposes. +func (n *node) print(w io.Writer, level int) { + fmt.Fprintf(w, "%sNODE:%v\n", strings.Repeat(" ", level), n.items) + for _, c := range n.children { + c.print(w, level+1) + } +} + +// BTree is an implementation of a B-Tree. +// +// BTree stores Item instances in an ordered structure, allowing easy insertion, +// removal, and iteration. +// +// Write operations are not safe for concurrent mutation by multiple +// goroutines, but Read operations are. +type BTree struct { + degree int + length int + root *node + cow *copyOnWriteContext +} + +// copyOnWriteContext pointers determine node ownership... a tree with a write +// context equivalent to a node's write context is allowed to modify that node. +// A tree whose write context does not match a node's is not allowed to modify +// it, and must create a new, writable copy (IE: it's a Clone). +// +// When doing any write operation, we maintain the invariant that the current +// node's context is equal to the context of the tree that requested the write. +// We do this by, before we descend into any node, creating a copy with the +// correct context if the contexts don't match. +// +// Since the node we're currently visiting on any write has the requesting +// tree's context, that node is modifiable in place. Children of that node may +// not share context, but before we descend into them, we'll make a mutable +// copy. +type copyOnWriteContext struct { + freelist *FreeList +} + +// Clone clones the btree, lazily. Clone should not be called concurrently, +// but the original tree (t) and the new tree (t2) can be used concurrently +// once the Clone call completes. +// +// The internal tree structure of b is marked read-only and shared between t and +// t2. Writes to both t and t2 use copy-on-write logic, creating new nodes +// whenever one of b's original nodes would have been modified. Read operations +// should have no performance degredation. Write operations for both t and t2 +// will initially experience minor slow-downs caused by additional allocs and +// copies due to the aforementioned copy-on-write logic, but should converge to +// the original performance characteristics of the original tree. +func (t *BTree) Clone() (t2 *BTree) { + // Create two entirely new copy-on-write contexts. + // This operation effectively creates three trees: + // the original, shared nodes (old b.cow) + // the new b.cow nodes + // the new out.cow nodes + cow1, cow2 := *t.cow, *t.cow + out := *t + t.cow = &cow1 + out.cow = &cow2 + return &out +} + +// maxItems returns the max number of items to allow per node. +func (t *BTree) maxItems() int { + return t.degree*2 - 1 +} + +// minItems returns the min number of items to allow per node (ignored for the +// root node). +func (t *BTree) minItems() int { + return t.degree - 1 +} + +func (c *copyOnWriteContext) newNode() (n *node) { + n = c.freelist.newNode() + n.cow = c + return +} + +func (c *copyOnWriteContext) freeNode(n *node) { + if n.cow == c { + // clear to allow GC + n.items.truncate(0) + n.children.truncate(0) + n.cow = nil + c.freelist.freeNode(n) + } +} + +// ReplaceOrInsert adds the given item to the tree. If an item in the tree +// already equals the given one, it is removed from the tree and returned. +// Otherwise, nil is returned. +// +// nil cannot be added to the tree (will panic). +func (t *BTree) ReplaceOrInsert(item Item) Item { + if item == nil { + panic("nil item being added to BTree") + } + if t.root == nil { + t.root = t.cow.newNode() + t.root.items = append(t.root.items, item) + t.length++ + return nil + } else { + t.root = t.root.mutableFor(t.cow) + if len(t.root.items) >= t.maxItems() { + item2, second := t.root.split(t.maxItems() / 2) + oldroot := t.root + t.root = t.cow.newNode() + t.root.items = append(t.root.items, item2) + t.root.children = append(t.root.children, oldroot, second) + } + } + out := t.root.insert(item, t.maxItems()) + if out == nil { + t.length++ + } + return out +} + +// Delete removes an item equal to the passed in item from the tree, returning +// it. If no such item exists, returns nil. +func (t *BTree) Delete(item Item) Item { + return t.deleteItem(item, removeItem) +} + +// DeleteMin removes the smallest item in the tree and returns it. +// If no such item exists, returns nil. +func (t *BTree) DeleteMin() Item { + return t.deleteItem(nil, removeMin) +} + +// DeleteMax removes the largest item in the tree and returns it. +// If no such item exists, returns nil. +func (t *BTree) DeleteMax() Item { + return t.deleteItem(nil, removeMax) +} + +func (t *BTree) deleteItem(item Item, typ toRemove) Item { + if t.root == nil || len(t.root.items) == 0 { + return nil + } + t.root = t.root.mutableFor(t.cow) + out := t.root.remove(item, t.minItems(), typ) + if len(t.root.items) == 0 && len(t.root.children) > 0 { + oldroot := t.root + t.root = t.root.children[0] + t.cow.freeNode(oldroot) + } + if out != nil { + t.length-- + } + return out +} + +// AscendRange calls the iterator for every value in the tree within the range +// [greaterOrEqual, lessThan), until iterator returns false. +func (t *BTree) AscendRange(greaterOrEqual, lessThan Item, iterator ItemIterator) { + if t.root == nil { + return + } + t.root.iterate(ascend, greaterOrEqual, lessThan, true, false, iterator) +} + +// AscendLessThan calls the iterator for every value in the tree within the range +// [first, pivot), until iterator returns false. +func (t *BTree) AscendLessThan(pivot Item, iterator ItemIterator) { + if t.root == nil { + return + } + t.root.iterate(ascend, nil, pivot, false, false, iterator) +} + +// AscendGreaterOrEqual calls the iterator for every value in the tree within +// the range [pivot, last], until iterator returns false. +func (t *BTree) AscendGreaterOrEqual(pivot Item, iterator ItemIterator) { + if t.root == nil { + return + } + t.root.iterate(ascend, pivot, nil, true, false, iterator) +} + +// Ascend calls the iterator for every value in the tree within the range +// [first, last], until iterator returns false. +func (t *BTree) Ascend(iterator ItemIterator) { + if t.root == nil { + return + } + t.root.iterate(ascend, nil, nil, false, false, iterator) +} + +// DescendRange calls the iterator for every value in the tree within the range +// [lessOrEqual, greaterThan), until iterator returns false. +func (t *BTree) DescendRange(lessOrEqual, greaterThan Item, iterator ItemIterator) { + if t.root == nil { + return + } + t.root.iterate(descend, lessOrEqual, greaterThan, true, false, iterator) +} + +// DescendLessOrEqual calls the iterator for every value in the tree within the range +// [pivot, first], until iterator returns false. +func (t *BTree) DescendLessOrEqual(pivot Item, iterator ItemIterator) { + if t.root == nil { + return + } + t.root.iterate(descend, pivot, nil, true, false, iterator) +} + +// DescendGreaterThan calls the iterator for every value in the tree within +// the range [last, pivot), until iterator returns false. +func (t *BTree) DescendGreaterThan(pivot Item, iterator ItemIterator) { + if t.root == nil { + return + } + t.root.iterate(descend, nil, pivot, false, false, iterator) +} + +// Descend calls the iterator for every value in the tree within the range +// [last, first], until iterator returns false. +func (t *BTree) Descend(iterator ItemIterator) { + if t.root == nil { + return + } + t.root.iterate(descend, nil, nil, false, false, iterator) +} + +// Get looks for the key item in the tree, returning it. It returns nil if +// unable to find that item. +func (t *BTree) Get(key Item) Item { + if t.root == nil { + return nil + } + return t.root.get(key) +} + +// Min returns the smallest item in the tree, or nil if the tree is empty. +func (t *BTree) Min() Item { + return min(t.root) +} + +// Max returns the largest item in the tree, or nil if the tree is empty. +func (t *BTree) Max() Item { + return max(t.root) +} + +// Has returns true if the given key is in the tree. +func (t *BTree) Has(key Item) bool { + return t.Get(key) != nil +} + +// Len returns the number of items currently in the tree. +func (t *BTree) Len() int { + return t.length +} + +// Int implements the Item interface for integers. +type Int int + +// Less returns true if int(a) < int(b). +func (a Int) Less(b Item) bool { + return a < b.(Int) +} \ No newline at end of file diff --git a/g/core/types/gbtree/gbtree_test.go b/g/core/types/gbtree/gbtree_test.go new file mode 100644 index 000000000..398989a81 --- /dev/null +++ b/g/core/types/gbtree/gbtree_test.go @@ -0,0 +1,676 @@ + +package gbtree + +import ( + "flag" + "fmt" + "math/rand" + "reflect" + "sort" + "sync" + "testing" + "time" +) + +func init() { + seed := time.Now().Unix() + fmt.Println(seed) + rand.Seed(seed) +} + +// perm returns a random permutation of n Int items in the range [0, n). +func perm(n int) (out []Item) { + for _, v := range rand.Perm(n) { + out = append(out, Int(v)) + } + return +} + +// rang returns an ordered list of Int items in the range [0, n). +func rang(n int) (out []Item) { + for i := 0; i < n; i++ { + out = append(out, Int(i)) + } + return +} + +// all extracts all items from a tree in order as a slice. +func all(t *BTree) (out []Item) { + t.Ascend(func(a Item) bool { + out = append(out, a) + return true + }) + return +} + +// rangerev returns a reversed ordered list of Int items in the range [0, n). +func rangrev(n int) (out []Item) { + for i := n - 1; i >= 0; i-- { + out = append(out, Int(i)) + } + return +} + +// allrev extracts all items from a tree in reverse order as a slice. +func allrev(t *BTree) (out []Item) { + t.Descend(func(a Item) bool { + out = append(out, a) + return true + }) + return +} + +var btreeDegree = flag.Int("degree", 32, "B-Tree degree") + +func TestBTree(t *testing.T) { + tr := New(*btreeDegree) + const treeSize = 10000 + for i := 0; i < 10; i++ { + if min := tr.Min(); min != nil { + t.Fatalf("empty min, got %+v", min) + } + if max := tr.Max(); max != nil { + t.Fatalf("empty max, got %+v", max) + } + for _, item := range perm(treeSize) { + if x := tr.ReplaceOrInsert(item); x != nil { + t.Fatal("insert found item", item) + } + } + for _, item := range perm(treeSize) { + if x := tr.ReplaceOrInsert(item); x == nil { + t.Fatal("insert didn't find item", item) + } + } + if min, want := tr.Min(), Item(Int(0)); min != want { + t.Fatalf("min: want %+v, got %+v", want, min) + } + if max, want := tr.Max(), Item(Int(treeSize-1)); max != want { + t.Fatalf("max: want %+v, got %+v", want, max) + } + got := all(tr) + want := rang(treeSize) + if !reflect.DeepEqual(got, want) { + t.Fatalf("mismatch:\n got: %v\nwant: %v", got, want) + } + + gotrev := allrev(tr) + wantrev := rangrev(treeSize) + if !reflect.DeepEqual(gotrev, wantrev) { + t.Fatalf("mismatch:\n got: %v\nwant: %v", got, want) + } + + for _, item := range perm(treeSize) { + if x := tr.Delete(item); x == nil { + t.Fatalf("didn't find %v", item) + } + } + if got = all(tr); len(got) > 0 { + t.Fatalf("some left!: %v", got) + } + } +} + +func ExampleBTree() { + tr := New(*btreeDegree) + for i := Int(0); i < 10; i++ { + tr.ReplaceOrInsert(i) + } + fmt.Println("len: ", tr.Len()) + fmt.Println("get3: ", tr.Get(Int(3))) + fmt.Println("get100: ", tr.Get(Int(100))) + fmt.Println("del4: ", tr.Delete(Int(4))) + fmt.Println("del100: ", tr.Delete(Int(100))) + fmt.Println("replace5: ", tr.ReplaceOrInsert(Int(5))) + fmt.Println("replace100:", tr.ReplaceOrInsert(Int(100))) + fmt.Println("min: ", tr.Min()) + fmt.Println("delmin: ", tr.DeleteMin()) + fmt.Println("max: ", tr.Max()) + fmt.Println("delmax: ", tr.DeleteMax()) + fmt.Println("len: ", tr.Len()) + // Output: + // len: 10 + // get3: 3 + // get100: + // del4: 4 + // del100: + // replace5: 5 + // replace100: + // min: 0 + // delmin: 0 + // max: 100 + // delmax: 100 + // len: 8 +} + +func TestDeleteMin(t *testing.T) { + tr := New(3) + for _, v := range perm(100) { + tr.ReplaceOrInsert(v) + } + var got []Item + for v := tr.DeleteMin(); v != nil; v = tr.DeleteMin() { + got = append(got, v) + } + if want := rang(100); !reflect.DeepEqual(got, want) { + t.Fatalf("ascendrange:\n got: %v\nwant: %v", got, want) + } +} + +func TestDeleteMax(t *testing.T) { + tr := New(3) + for _, v := range perm(100) { + tr.ReplaceOrInsert(v) + } + var got []Item + for v := tr.DeleteMax(); v != nil; v = tr.DeleteMax() { + got = append(got, v) + } + // Reverse our list. + for i := 0; i < len(got)/2; i++ { + got[i], got[len(got)-i-1] = got[len(got)-i-1], got[i] + } + if want := rang(100); !reflect.DeepEqual(got, want) { + t.Fatalf("ascendrange:\n got: %v\nwant: %v", got, want) + } +} + +func TestAscendRange(t *testing.T) { + tr := New(2) + for _, v := range perm(100) { + tr.ReplaceOrInsert(v) + } + var got []Item + tr.AscendRange(Int(40), Int(60), func(a Item) bool { + got = append(got, a) + return true + }) + if want := rang(100)[40:60]; !reflect.DeepEqual(got, want) { + t.Fatalf("ascendrange:\n got: %v\nwant: %v", got, want) + } + got = got[:0] + tr.AscendRange(Int(40), Int(60), func(a Item) bool { + if a.(Int) > 50 { + return false + } + got = append(got, a) + return true + }) + if want := rang(100)[40:51]; !reflect.DeepEqual(got, want) { + t.Fatalf("ascendrange:\n got: %v\nwant: %v", got, want) + } +} + +func TestDescendRange(t *testing.T) { + tr := New(2) + for _, v := range perm(100) { + tr.ReplaceOrInsert(v) + } + var got []Item + tr.DescendRange(Int(60), Int(40), func(a Item) bool { + got = append(got, a) + return true + }) + if want := rangrev(100)[39:59]; !reflect.DeepEqual(got, want) { + t.Fatalf("descendrange:\n got: %v\nwant: %v", got, want) + } + got = got[:0] + tr.DescendRange(Int(60), Int(40), func(a Item) bool { + if a.(Int) < 50 { + return false + } + got = append(got, a) + return true + }) + if want := rangrev(100)[39:50]; !reflect.DeepEqual(got, want) { + t.Fatalf("descendrange:\n got: %v\nwant: %v", got, want) + } +} +func TestAscendLessThan(t *testing.T) { + tr := New(*btreeDegree) + for _, v := range perm(100) { + tr.ReplaceOrInsert(v) + } + var got []Item + tr.AscendLessThan(Int(60), func(a Item) bool { + got = append(got, a) + return true + }) + if want := rang(100)[:60]; !reflect.DeepEqual(got, want) { + t.Fatalf("ascendrange:\n got: %v\nwant: %v", got, want) + } + got = got[:0] + tr.AscendLessThan(Int(60), func(a Item) bool { + if a.(Int) > 50 { + return false + } + got = append(got, a) + return true + }) + if want := rang(100)[:51]; !reflect.DeepEqual(got, want) { + t.Fatalf("ascendrange:\n got: %v\nwant: %v", got, want) + } +} + +func TestDescendLessOrEqual(t *testing.T) { + tr := New(*btreeDegree) + for _, v := range perm(100) { + tr.ReplaceOrInsert(v) + } + var got []Item + tr.DescendLessOrEqual(Int(40), func(a Item) bool { + got = append(got, a) + return true + }) + if want := rangrev(100)[59:]; !reflect.DeepEqual(got, want) { + t.Fatalf("descendlessorequal:\n got: %v\nwant: %v", got, want) + } + got = got[:0] + tr.DescendLessOrEqual(Int(60), func(a Item) bool { + if a.(Int) < 50 { + return false + } + got = append(got, a) + return true + }) + if want := rangrev(100)[39:50]; !reflect.DeepEqual(got, want) { + t.Fatalf("descendlessorequal:\n got: %v\nwant: %v", got, want) + } +} +func TestAscendGreaterOrEqual(t *testing.T) { + tr := New(*btreeDegree) + for _, v := range perm(100) { + tr.ReplaceOrInsert(v) + } + var got []Item + tr.AscendGreaterOrEqual(Int(40), func(a Item) bool { + got = append(got, a) + return true + }) + if want := rang(100)[40:]; !reflect.DeepEqual(got, want) { + t.Fatalf("ascendrange:\n got: %v\nwant: %v", got, want) + } + got = got[:0] + tr.AscendGreaterOrEqual(Int(40), func(a Item) bool { + if a.(Int) > 50 { + return false + } + got = append(got, a) + return true + }) + if want := rang(100)[40:51]; !reflect.DeepEqual(got, want) { + t.Fatalf("ascendrange:\n got: %v\nwant: %v", got, want) + } +} + +func TestDescendGreaterThan(t *testing.T) { + tr := New(*btreeDegree) + for _, v := range perm(100) { + tr.ReplaceOrInsert(v) + } + var got []Item + tr.DescendGreaterThan(Int(40), func(a Item) bool { + got = append(got, a) + return true + }) + if want := rangrev(100)[:59]; !reflect.DeepEqual(got, want) { + t.Fatalf("descendgreaterthan:\n got: %v\nwant: %v", got, want) + } + got = got[:0] + tr.DescendGreaterThan(Int(40), func(a Item) bool { + if a.(Int) < 50 { + return false + } + got = append(got, a) + return true + }) + if want := rangrev(100)[:50]; !reflect.DeepEqual(got, want) { + t.Fatalf("descendgreaterthan:\n got: %v\nwant: %v", got, want) + } +} + +const benchmarkTreeSize = 10000 + +func BenchmarkInsert(b *testing.B) { + b.StopTimer() + insertP := perm(benchmarkTreeSize) + b.StartTimer() + i := 0 + for i < b.N { + tr := New(*btreeDegree) + for _, item := range insertP { + tr.ReplaceOrInsert(item) + i++ + if i >= b.N { + return + } + } + } +} + +func BenchmarkDeleteInsert(b *testing.B) { + b.StopTimer() + insertP := perm(benchmarkTreeSize) + tr := New(*btreeDegree) + for _, item := range insertP { + tr.ReplaceOrInsert(item) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + tr.Delete(insertP[i%benchmarkTreeSize]) + tr.ReplaceOrInsert(insertP[i%benchmarkTreeSize]) + } +} + +func BenchmarkDeleteInsertCloneOnce(b *testing.B) { + b.StopTimer() + insertP := perm(benchmarkTreeSize) + tr := New(*btreeDegree) + for _, item := range insertP { + tr.ReplaceOrInsert(item) + } + tr = tr.Clone() + b.StartTimer() + for i := 0; i < b.N; i++ { + tr.Delete(insertP[i%benchmarkTreeSize]) + tr.ReplaceOrInsert(insertP[i%benchmarkTreeSize]) + } +} + +func BenchmarkDeleteInsertCloneEachTime(b *testing.B) { + b.StopTimer() + insertP := perm(benchmarkTreeSize) + tr := New(*btreeDegree) + for _, item := range insertP { + tr.ReplaceOrInsert(item) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + tr = tr.Clone() + tr.Delete(insertP[i%benchmarkTreeSize]) + tr.ReplaceOrInsert(insertP[i%benchmarkTreeSize]) + } +} + +func BenchmarkDelete(b *testing.B) { + b.StopTimer() + insertP := perm(benchmarkTreeSize) + removeP := perm(benchmarkTreeSize) + b.StartTimer() + i := 0 + for i < b.N { + b.StopTimer() + tr := New(*btreeDegree) + for _, v := range insertP { + tr.ReplaceOrInsert(v) + } + b.StartTimer() + for _, item := range removeP { + tr.Delete(item) + i++ + if i >= b.N { + return + } + } + if tr.Len() > 0 { + panic(tr.Len()) + } + } +} + +func BenchmarkGet(b *testing.B) { + b.StopTimer() + insertP := perm(benchmarkTreeSize) + removeP := perm(benchmarkTreeSize) + b.StartTimer() + i := 0 + for i < b.N { + b.StopTimer() + tr := New(*btreeDegree) + for _, v := range insertP { + tr.ReplaceOrInsert(v) + } + b.StartTimer() + for _, item := range removeP { + tr.Get(item) + i++ + if i >= b.N { + return + } + } + } +} + +func BenchmarkGetCloneEachTime(b *testing.B) { + b.StopTimer() + insertP := perm(benchmarkTreeSize) + removeP := perm(benchmarkTreeSize) + b.StartTimer() + i := 0 + for i < b.N { + b.StopTimer() + tr := New(*btreeDegree) + for _, v := range insertP { + tr.ReplaceOrInsert(v) + } + b.StartTimer() + for _, item := range removeP { + tr = tr.Clone() + tr.Get(item) + i++ + if i >= b.N { + return + } + } + } +} + +type byInts []Item + +func (a byInts) Len() int { + return len(a) +} + +func (a byInts) Less(i, j int) bool { + return a[i].(Int) < a[j].(Int) +} + +func (a byInts) Swap(i, j int) { + a[i], a[j] = a[j], a[i] +} + +func BenchmarkAscend(b *testing.B) { + arr := perm(benchmarkTreeSize) + tr := New(*btreeDegree) + for _, v := range arr { + tr.ReplaceOrInsert(v) + } + sort.Sort(byInts(arr)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + j := 0 + tr.Ascend(func(item Item) bool { + if item.(Int) != arr[j].(Int) { + b.Fatalf("mismatch: expected: %v, got %v", arr[j].(Int), item.(Int)) + } + j++ + return true + }) + } +} + +func BenchmarkDescend(b *testing.B) { + arr := perm(benchmarkTreeSize) + tr := New(*btreeDegree) + for _, v := range arr { + tr.ReplaceOrInsert(v) + } + sort.Sort(byInts(arr)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + j := len(arr) - 1 + tr.Descend(func(item Item) bool { + if item.(Int) != arr[j].(Int) { + b.Fatalf("mismatch: expected: %v, got %v", arr[j].(Int), item.(Int)) + } + j-- + return true + }) + } +} +func BenchmarkAscendRange(b *testing.B) { + arr := perm(benchmarkTreeSize) + tr := New(*btreeDegree) + for _, v := range arr { + tr.ReplaceOrInsert(v) + } + sort.Sort(byInts(arr)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + j := 100 + tr.AscendRange(Int(100), arr[len(arr)-100], func(item Item) bool { + if item.(Int) != arr[j].(Int) { + b.Fatalf("mismatch: expected: %v, got %v", arr[j].(Int), item.(Int)) + } + j++ + return true + }) + if j != len(arr)-100 { + b.Fatalf("expected: %v, got %v", len(arr)-100, j) + } + } +} + +func BenchmarkDescendRange(b *testing.B) { + arr := perm(benchmarkTreeSize) + tr := New(*btreeDegree) + for _, v := range arr { + tr.ReplaceOrInsert(v) + } + sort.Sort(byInts(arr)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + j := len(arr) - 100 + tr.DescendRange(arr[len(arr)-100], Int(100), func(item Item) bool { + if item.(Int) != arr[j].(Int) { + b.Fatalf("mismatch: expected: %v, got %v", arr[j].(Int), item.(Int)) + } + j-- + return true + }) + if j != 100 { + b.Fatalf("expected: %v, got %v", len(arr)-100, j) + } + } +} +func BenchmarkAscendGreaterOrEqual(b *testing.B) { + arr := perm(benchmarkTreeSize) + tr := New(*btreeDegree) + for _, v := range arr { + tr.ReplaceOrInsert(v) + } + sort.Sort(byInts(arr)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + j := 100 + k := 0 + tr.AscendGreaterOrEqual(Int(100), func(item Item) bool { + if item.(Int) != arr[j].(Int) { + b.Fatalf("mismatch: expected: %v, got %v", arr[j].(Int), item.(Int)) + } + j++ + k++ + return true + }) + if j != len(arr) { + b.Fatalf("expected: %v, got %v", len(arr), j) + } + if k != len(arr)-100 { + b.Fatalf("expected: %v, got %v", len(arr)-100, k) + } + } +} +func BenchmarkDescendLessOrEqual(b *testing.B) { + arr := perm(benchmarkTreeSize) + tr := New(*btreeDegree) + for _, v := range arr { + tr.ReplaceOrInsert(v) + } + sort.Sort(byInts(arr)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + j := len(arr) - 100 + k := len(arr) + tr.DescendLessOrEqual(arr[len(arr)-100], func(item Item) bool { + if item.(Int) != arr[j].(Int) { + b.Fatalf("mismatch: expected: %v, got %v", arr[j].(Int), item.(Int)) + } + j-- + k-- + return true + }) + if j != -1 { + b.Fatalf("expected: %v, got %v", -1, j) + } + if k != 99 { + b.Fatalf("expected: %v, got %v", 99, k) + } + } +} + +const cloneTestSize = 10000 + +func cloneTest(t *testing.T, b *BTree, start int, p []Item, wg *sync.WaitGroup, trees *[]*BTree) { + t.Logf("Starting new clone at %v", start) + *trees = append(*trees, b) + for i := start; i < cloneTestSize; i++ { + b.ReplaceOrInsert(p[i]) + if i%(cloneTestSize/5) == 0 { + wg.Add(1) + go cloneTest(t, b.Clone(), i+1, p, wg, trees) + } + } + wg.Done() +} + +func TestCloneConcurrentOperations(t *testing.T) { + b := New(*btreeDegree) + trees := []*BTree{} + p := perm(cloneTestSize) + var wg sync.WaitGroup + wg.Add(1) + go cloneTest(t, b, 0, p, &wg, &trees) + wg.Wait() + want := rang(cloneTestSize) + t.Logf("Starting equality checks on %d trees", len(trees)) + for i, tree := range trees { + if !reflect.DeepEqual(want, all(tree)) { + t.Errorf("tree %v mismatch", i) + } + } + t.Log("Removing half from first half") + toRemove := rang(cloneTestSize)[cloneTestSize/2:] + for i := 0; i < len(trees)/2; i++ { + tree := trees[i] + wg.Add(1) + go func() { + for _, item := range toRemove { + tree.Delete(item) + } + wg.Done() + }() + } + wg.Wait() + t.Log("Checking all values again") + for i, tree := range trees { + var wantpart []Item + if i < len(trees)/2 { + wantpart = want[:cloneTestSize/2] + } else { + wantpart = want + } + if got := all(tree); !reflect.DeepEqual(wantpart, got) { + t.Errorf("tree %v mismatch, want %v got %v", i, len(want), len(got)) + } + } +} \ No newline at end of file diff --git a/g/core/types/glist/safelist.go b/g/core/types/glist/safelist.go new file mode 100644 index 000000000..4e48cbea4 --- /dev/null +++ b/g/core/types/glist/safelist.go @@ -0,0 +1,209 @@ +package glist + +import ( + "container/list" + "sync" +) + +// 变长双向链表 +type SafeList struct { + sync.RWMutex + L *list.List +} + +// 获得一个变长链表指针 +func NewSafeList() *SafeList { + return &SafeList{L: list.New()} +} + +// 往链表头入栈数据项 +func (this *SafeList) PushFront(v interface{}) *list.Element { + this.Lock() + e := this.L.PushFront(v) + this.Unlock() + return e +} + +// 往链表尾入栈数据项 +func (this *SafeList) PushBack(v interface{}) *list.Element { + this.Lock() + r := this.L.PushBack(v) + this.Unlock() + return r +} + +// 在list 中元素mark之后插入一个值为v的元素，并返回该元素，如果mark不是list中元素，则list不改变。 +func (this *SafeList) InsertAfter(v interface{}, mark *list.Element) *list.Element { + this.Lock() + r := this.L.InsertAfter(v, mark) + this.Unlock() + return r +} + +// 在list 中元素mark之前插入一个值为v的元素，并返回该元素，如果mark不是list中元素，则list不改变。 +func (this *SafeList) InsertBefore(v interface{}, mark *list.Element) *list.Element { + this.Lock() + r := this.L.InsertBefore(v, mark) + this.Unlock() + return r +} + + +// 批量往链表头入栈数据项 +func (this *SafeList) BatchPushFront(vs []interface{}) { + this.Lock() + for _, item := range vs { + this.L.PushFront(item) + } + this.Unlock() +} + +// 从链表尾端出栈数据项(删除) +func (this *SafeList) PopBack() interface{} { + this.Lock() + if elem := this.L.Back(); elem != nil { + item := this.L.Remove(elem) + this.Unlock() + return item + } + this.Unlock() + return nil +} + +// 批量从链表尾端出栈数据项(删除) +func (this *SafeList) BatchPopBack(max int) []interface{} { + this.Lock() + count := this.L.Len() + if count == 0 { + this.Unlock() + return []interface{}{} + } + + if count > max { + count = max + } + items := make([]interface{}, 0, count) + for i := 0; i < count; i++ { + item := this.L.Remove(this.L.Back()) + items = append(items, item) + } + this.Unlock() + return items +} + +// 批量从链表尾端依次获取所有数据 +func (this *SafeList) PopBackAll() []interface{} { + this.Lock() + + count := this.L.Len() + if count == 0 { + this.Unlock() + return []interface{}{} + } + + items := make([]interface{}, 0, count) + for i := 0; i < count; i++ { + item := this.L.Remove(this.L.Back()) + items = append(items, item) + } + + this.Unlock() + return items +} + +// 删除数据项 +func (this *SafeList) Remove(e *list.Element) interface{} { + this.Lock() + r := this.L.Remove(e) + this.Unlock() + return r +} + +// 删除所有数据项 +func (this *SafeList) RemoveAll() { + this.Lock() + this.L = list.New() + this.Unlock() +} + +// 从链表头获取所有数据(不删除) +func (this *SafeList) FrontAll() []interface{} { + this.RLock() + count := this.L.Len() + if count == 0 { + this.RUnlock() + return []interface{}{} + } + + items := make([]interface{}, 0, count) + for e := this.L.Front(); e != nil; e = e.Next() { + items = append(items, e.Value) + } + this.RUnlock() + return items +} + +// 从链表尾获取所有数据(不删除) +func (this *SafeList) BackAll() []interface{} { + this.RLock() + count := this.L.Len() + if count == 0 { + this.RUnlock() + return []interface{}{} + } + + items := make([]interface{}, 0, count) + for e := this.L.Back(); e != nil; e = e.Prev() { + items = append(items, e.Value) + } + this.RUnlock() + return items +} + +// 获取链表头值(不删除) +func (this *SafeList) FrontItem() interface{} { + this.RLock() + if f := this.L.Front(); f != nil { + this.RUnlock() + return f.Value + } + + this.RUnlock() + return nil +} + +// 获取链表尾值(不删除) +func (this *SafeList) BackItem() interface{} { + this.RLock() + if f := this.L.Back(); f != nil { + this.RUnlock() + return f.Value + } + + this.RUnlock() + return nil +} + +// 获取表头指针 +func (this *SafeList) Front() *list.Element { + this.RLock() + r := this.L.Front() + this.RUnlock() + return r +} + +// 获取表位指针 +func (this *SafeList) Back() *list.Element { + this.RLock() + r := this.L.Back() + this.RUnlock() + return r +} + +// 获取链表长度 +func (this *SafeList) Len() int { + this.RLock() + length := this.L.Len() + this.RUnlock() + return length +} diff --git a/g/core/types/gmap/int_bool_map.go b/g/core/types/gmap/int_bool_map.go new file mode 100644 index 000000000..5756c2ff6 --- /dev/null +++ b/g/core/types/gmap/int_bool_map.go @@ -0,0 +1,132 @@ +package gmap + +import ( + "sync" +) + +type IntBoolMap struct { + sync.RWMutex + m map[int]bool +} + +func NewIntBoolMap() *IntBoolMap { + return &IntBoolMap{ + m: make(map[int]bool), + } +} + +// 哈希表克隆 +func (this *IntBoolMap) Clone() *map[int]bool { + m := make(map[int]bool) + this.RLock() + for k, v := range this.m { + m[k] = v + } + this.RUnlock() + return &m +} + +// 设置键值对 +func (this *IntBoolMap) Set(key int, val bool) { + this.Lock() + this.m[key] = val + this.Unlock() +} + +// 批量设置键值对 +func (this *IntBoolMap) BatchSet(m map[int]bool) { + this.Lock() + for k, v := range m { + this.m[k] = v + } + this.Unlock() +} + +// 获取键值 +func (this *IntBoolMap) Get(key int) (bool) { + this.RLock() + val, _ := this.m[key] + this.RUnlock() + return val +} + +// 删除键值对 +func (this *IntBoolMap) Remove(key int) { + this.Lock() + delete(this.m, key) + this.Unlock() +} + +// 批量删除键值对 +func (this *IntBoolMap) BatchRemove(keys []int) { + this.Lock() + for _, key := range keys { + delete(this.m, key) + } + this.Unlock() +} + +// 返回对应的键值，并删除该键值 +func (this *IntBoolMap) GetAndRemove(key int) (bool) { + this.Lock() + val, exists := this.m[key] + if exists { + delete(this.m, key) + } + this.Unlock() + return val +} + +// 返回键列表 +func (this *IntBoolMap) Keys() []int { + this.RLock() + keys := make([]int, 0) + for key, _ := range this.m { + keys = append(keys, key) + } + this.RUnlock() + return keys +} + +// 返回值列表(注意是随机排序) +//func (this *IntBoolMap) Values() []bool { +// this.RLock() +// vals := make([]bool, 0) +// for _, val := range this.m { +// vals = append(vals, val) +// } +// this.RUnlock() +// return vals +//} + +// 是否存在某个键 +func (this *IntBoolMap) Contains(key int) bool { + this.RLock() + _, exists := this.m[key] + this.RUnlock() + return exists +} + +// 哈希表大小 +func (this *IntBoolMap) Size() int { + this.RLock() + len := len(this.m) + this.RUnlock() + return len +} + +// 哈希表是否为空 +func (this *IntBoolMap) IsEmpty() bool { + this.RLock() + empty := (len(this.m) == 0) + this.RUnlock() + return empty +} + +// 清空哈希表 +func (this *IntBoolMap) Clear() { + this.Lock() + this.m = make(map[int]bool) + this.Unlock() +} + diff --git a/g/core/types/gmap/int_int_map.go b/g/core/types/gmap/int_int_map.go new file mode 100644 index 000000000..8b2de2af1 --- /dev/null +++ b/g/core/types/gmap/int_int_map.go @@ -0,0 +1,132 @@ +package gmap + +import ( + "sync" +) + +type IntIntMap struct { + sync.RWMutex + m map[int]int +} + +func NewIntIntMap() *IntIntMap { + return &IntIntMap{ + m: make(map[int]int), + } +} + +// 哈希表克隆 +func (this *IntIntMap) Clone() *map[int]int { + m := make(map[int]int) + this.RLock() + for k, v := range this.m { + m[k] = v + } + this.RUnlock() + return &m +} + +// 设置键值对 +func (this *IntIntMap) Set(key int, val int) { + this.Lock() + this.m[key] = val + this.Unlock() +} + +// 批量设置键值对 +func (this *IntIntMap) BatchSet(m map[int]int) { + this.Lock() + for k, v := range m { + this.m[k] = v + } + this.Unlock() +} + +// 获取键值 +func (this *IntIntMap) Get(key int) (int) { + this.RLock() + val, _ := this.m[key] + this.RUnlock() + return val +} + +// 删除键值对 +func (this *IntIntMap) Remove(key int) { + this.Lock() + delete(this.m, key) + this.Unlock() +} + +// 批量删除键值对 +func (this *IntIntMap) BatchRemove(keys []int) { + this.Lock() + for _, key := range keys { + delete(this.m, key) + } + this.Unlock() +} + +// 返回对应的键值，并删除该键值 +func (this *IntIntMap) GetAndRemove(key int) (int) { + this.Lock() + val, exists := this.m[key] + if exists { + delete(this.m, key) + } + this.Unlock() + return val +} + +// 返回键列表 +func (this *IntIntMap) Keys() []int { + this.RLock() + keys := make([]int, 0) + for key, _ := range this.m { + keys = append(keys, key) + } + this.RUnlock() + return keys +} + +// 返回值列表(注意是随机排序) +func (this *IntIntMap) Values() []int { + this.RLock() + vals := make([]int, 0) + for _, val := range this.m { + vals = append(vals, val) + } + this.RUnlock() + return vals +} + +// 是否存在某个键 +func (this *IntIntMap) Contains(key int) bool { + this.RLock() + _, exists := this.m[key] + this.RUnlock() + return exists +} + +// 哈希表大小 +func (this *IntIntMap) Size() int { + this.RLock() + len := len(this.m) + this.RUnlock() + return len +} + +// 哈希表是否为空 +func (this *IntIntMap) IsEmpty() bool { + this.RLock() + empty := (len(this.m) == 0) + this.RUnlock() + return empty +} + +// 清空哈希表 +func (this *IntIntMap) Clear() { + this.Lock() + this.m = make(map[int]int) + this.Unlock() +} + diff --git a/g/core/types/gmap/int_interface_map.go b/g/core/types/gmap/int_interface_map.go new file mode 100644 index 000000000..989637167 --- /dev/null +++ b/g/core/types/gmap/int_interface_map.go @@ -0,0 +1,132 @@ +package gmap + +import ( + "sync" +) + +type IntInterfaceMap struct { + sync.RWMutex + m map[int]interface{} +} + +func NewIntInterfaceMap() *IntInterfaceMap { + return &IntInterfaceMap{ + m: make(map[int]interface{}), + } +} + +// 哈希表克隆 +func (this *IntInterfaceMap) Clone() *map[int]interface{} { + m := make(map[int]interface{}) + this.RLock() + for k, v := range this.m { + m[k] = v + } + this.RUnlock() + return &m +} + +// 设置键值对 +func (this *IntInterfaceMap) Set(key int, val interface{}) { + this.Lock() + this.m[key] = val + this.Unlock() +} + +// 批量设置键值对 +func (this *IntInterfaceMap) BatchSet(m map[int]interface{}) { + this.Lock() + for k, v := range m { + this.m[k] = v + } + this.Unlock() +} + +// 获取键值 +func (this *IntInterfaceMap) Get(key int) (interface{}) { + this.RLock() + val, _ := this.m[key] + this.RUnlock() + return val +} + +// 删除键值对 +func (this *IntInterfaceMap) Remove(key int) { + this.Lock() + delete(this.m, key) + this.Unlock() +} + +// 批量删除键值对 +func (this *IntInterfaceMap) BatchRemove(keys []int) { + this.Lock() + for _, key := range keys { + delete(this.m, key) + } + this.Unlock() +} + +// 返回对应的键值，并删除该键值 +func (this *IntInterfaceMap) GetAndRemove(key int) (interface{}) { + this.Lock() + val, exists := this.m[key] + if exists { + delete(this.m, key) + } + this.Unlock() + return val +} + +// 返回键列表 +func (this *IntInterfaceMap) Keys() []int { + this.RLock() + keys := make([]int, 0) + for key, _ := range this.m { + keys = append(keys, key) + } + this.RUnlock() + return keys +} + +// 返回值列表(注意是随机排序) +func (this *IntInterfaceMap) Values() []interface{} { + this.RLock() + vals := make([]interface{}, 0) + for _, val := range this.m { + vals = append(vals, val) + } + this.RUnlock() + return vals +} + +// 是否存在某个键 +func (this *IntInterfaceMap) Contains(key int) bool { + this.RLock() + _, exists := this.m[key] + this.RUnlock() + return exists +} + +// 哈希表大小 +func (this *IntInterfaceMap) Size() int { + this.RLock() + len := len(this.m) + this.RUnlock() + return len +} + +// 哈希表是否为空 +func (this *IntInterfaceMap) IsEmpty() bool { + this.RLock() + empty := (len(this.m) == 0) + this.RUnlock() + return empty +} + +// 清空哈希表 +func (this *IntInterfaceMap) Clear() { + this.Lock() + this.m = make(map[int]interface{}) + this.Unlock() +} + diff --git a/g/core/types/gmap/int_string_map.go b/g/core/types/gmap/int_string_map.go new file mode 100644 index 000000000..a0a3d774a --- /dev/null +++ b/g/core/types/gmap/int_string_map.go @@ -0,0 +1,132 @@ +package gmap + +import ( + "sync" +) + +type IntStringMap struct { + sync.RWMutex + m map[int]string +} + +func NewIntStringMap() *IntStringMap { + return &IntStringMap{ + m: make(map[int]string), + } +} + +// 哈希表克隆 +func (this *IntStringMap) Clone() *map[int]string { + m := make(map[int]string) + this.RLock() + for k, v := range this.m { + m[k] = v + } + this.RUnlock() + return &m +} + +// 设置键值对 +func (this *IntStringMap) Set(key int, val string) { + this.Lock() + this.m[key] = val + this.Unlock() +} + +// 批量设置键值对 +func (this *IntStringMap) BatchSet(m map[int]string) { + this.Lock() + for k, v := range m { + this.m[k] = v + } + this.Unlock() +} + +// 获取键值 +func (this *IntStringMap) Get(key int) (string) { + this.RLock() + val, _ := this.m[key] + this.RUnlock() + return val +} + +// 删除键值对 +func (this *IntStringMap) Remove(key int) { + this.Lock() + delete(this.m, key) + this.Unlock() +} + +// 批量删除键值对 +func (this *IntStringMap) BatchRemove(keys []int) { + this.Lock() + for _, key := range keys { + delete(this.m, key) + } + this.Unlock() +} + +// 返回对应的键值，并删除该键值 +func (this *IntStringMap) GetAndRemove(key int) (string) { + this.Lock() + val, exists := this.m[key] + if exists { + delete(this.m, key) + } + this.Unlock() + return val +} + +// 返回键列表 +func (this *IntStringMap) Keys() []int { + this.RLock() + keys := make([]int, 0) + for key, _ := range this.m { + keys = append(keys, key) + } + this.RUnlock() + return keys +} + +// 返回值列表(注意是随机排序) +func (this *IntStringMap) Values() []string { + this.RLock() + vals := make([]string, 0) + for _, val := range this.m { + vals = append(vals, val) + } + this.RUnlock() + return vals +} + +// 是否存在某个键 +func (this *IntStringMap) Contains(key int) bool { + this.RLock() + _, exists := this.m[key] + this.RUnlock() + return exists +} + +// 哈希表大小 +func (this *IntStringMap) Size() int { + this.RLock() + len := len(this.m) + this.RUnlock() + return len +} + +// 哈希表是否为空 +func (this *IntStringMap) IsEmpty() bool { + this.RLock() + empty := (len(this.m) == 0) + this.RUnlock() + return empty +} + +// 清空哈希表 +func (this *IntStringMap) Clear() { + this.Lock() + this.m = make(map[int]string) + this.Unlock() +} + diff --git a/g/core/types/gmap/interface_interface_map.go b/g/core/types/gmap/interface_interface_map.go new file mode 100644 index 000000000..5a06c6e42 --- /dev/null +++ b/g/core/types/gmap/interface_interface_map.go @@ -0,0 +1,131 @@ +package gmap + +import ( + "sync" +) + +type InterfaceInterfaceMap struct { + sync.RWMutex + m map[interface{}]interface{} +} + +func NewInterfaceInterfaceMap() *InterfaceInterfaceMap { + return &InterfaceInterfaceMap{ + m: make(map[interface{}]interface{}), + } +} + +// 哈希表克隆 +func (this *InterfaceInterfaceMap) Clone() *map[interface{}]interface{} { + m := make(map[interface{}]interface{}) + this.RLock() + for k, v := range this.m { + m[k] = v + } + this.RUnlock() + return &m +} + +// 设置键值对 +func (this *InterfaceInterfaceMap) Set(key interface{}, val interface{}) { + this.Lock() + this.m[key] = val + this.Unlock() +} + +// 批量设置键值对 +func (this *InterfaceInterfaceMap) BatchSet(m map[interface{}]interface{}) { + this.Lock() + for k, v := range m { + this.m[k] = v + } + this.Unlock() +} + +// 获取键值 +func (this *InterfaceInterfaceMap) Get(key interface{}) (interface{}) { + this.RLock() + val, _ := this.m[key] + this.RUnlock() + return val +} + +// 删除键值对 +func (this *InterfaceInterfaceMap) Remove(key interface{}) { + this.Lock() + delete(this.m, key) + this.Unlock() +} + +// 批量删除键值对 +func (this *InterfaceInterfaceMap) BatchRemove(keys []interface{}) { + this.Lock() + for _, key := range keys { + delete(this.m, key) + } + this.Unlock() +} + +// 返回对应的键值，并删除该键值 +func (this *InterfaceInterfaceMap) GetAndRemove(key interface{}) (interface{}) { + this.Lock() + val, exists := this.m[key] + if exists { + delete(this.m, key) + } + this.Unlock() + return val +} + +// 返回键列表 +func (this *InterfaceInterfaceMap) Keys() []interface{} { + this.RLock() + keys := make([]interface{}, 0) + for key, _ := range this.m { + keys = append(keys, key) + } + this.RUnlock() + return keys +} + +// 返回值列表(注意是随机排序) +func (this *InterfaceInterfaceMap) Values() []interface{} { + this.RLock() + vals := make([]interface{}, 0) + for _, val := range this.m { + vals = append(vals, val) + } + this.RUnlock() + return vals +} + +// 是否存在某个键 +func (this *InterfaceInterfaceMap) Contains(key interface{}) bool { + this.RLock() + _, exists := this.m[key] + this.RUnlock() + return exists +} + +// 哈希表大小 +func (this *InterfaceInterfaceMap) Size() int { + this.RLock() + len := len(this.m) + this.RUnlock() + return len +} + +// 哈希表是否为空 +func (this *InterfaceInterfaceMap) IsEmpty() bool { + this.RLock() + empty := (len(this.m) == 0) + this.RUnlock() + return empty +} + +// 清空哈希表 +func (this *InterfaceInterfaceMap) Clear() { + this.Lock() + this.m = make(map[interface{}]interface{}) + this.Unlock() +} diff --git a/g/core/types/gmap/string_bool_map.go b/g/core/types/gmap/string_bool_map.go new file mode 100644 index 000000000..9aa08a9c6 --- /dev/null +++ b/g/core/types/gmap/string_bool_map.go @@ -0,0 +1,131 @@ +package gmap + +import ( + "sync" +) + +type StringBoolMap struct { + sync.RWMutex + m map[string]bool +} + +func NewStringBoolMap() *StringBoolMap { + return &StringBoolMap{ + m: make(map[string]bool), + } +} + +// 哈希表克隆 +func (this *StringBoolMap) Clone() *map[string]bool { + m := make(map[string]bool) + this.RLock() + for k, v := range this.m { + m[k] = v + } + this.RUnlock() + return &m +} + +// 设置键值对 +func (this *StringBoolMap) Set(key string, val bool) { + this.Lock() + this.m[key] = val + this.Unlock() +} + +// 批量设置键值对 +func (this *StringBoolMap) BatchSet(m map[string]bool) { + this.Lock() + for k, v := range m { + this.m[k] = v + } + this.Unlock() +} + +// 获取键值 +func (this *StringBoolMap) Get(key string) (bool) { + this.RLock() + val, _ := this.m[key] + this.RUnlock() + return val +} + +// 删除键值对 +func (this *StringBoolMap) Remove(key string) { + this.Lock() + delete(this.m, key) + this.Unlock() +} + +// 批量删除键值对 +func (this *StringBoolMap) BatchRemove(keys []string) { + this.Lock() + for _, key := range keys { + delete(this.m, key) + } + this.Unlock() +} + +// 返回对应的键值，并删除该键值 +func (this *StringBoolMap) GetAndRemove(key string) (bool) { + this.Lock() + val, exists := this.m[key] + if exists { + delete(this.m, key) + } + this.Unlock() + return val +} + +// 返回键列表 +func (this *StringBoolMap) Keys() []string { + this.RLock() + keys := make([]string, 0) + for key, _ := range this.m { + keys = append(keys, key) + } + this.RUnlock() + return keys +} + +// 返回值列表(注意是随机排序) +//func (this *StringBoolMap) Values() []bool { +// this.RLock() +// vals := make([]bool, 0) +// for _, val := range this.m { +// vals = append(vals, val) +// } +// this.RUnlock() +// return vals +//} + +// 是否存在某个键 +func (this *StringBoolMap) Contains(key string) bool { + this.RLock() + _, exists := this.m[key] + this.RUnlock() + return exists +} + +// 哈希表大小 +func (this *StringBoolMap) Size() int { + this.RLock() + len := len(this.m) + this.RUnlock() + return len +} + +// 哈希表是否为空 +func (this *StringBoolMap) IsEmpty() bool { + this.RLock() + empty := (len(this.m) == 0) + this.RUnlock() + return empty +} + +// 清空哈希表 +func (this *StringBoolMap) Clear() { + this.Lock() + this.m = make(map[string]bool) + this.Unlock() +} diff --git a/g/core/types/gmap/string_int_map.go b/g/core/types/gmap/string_int_map.go new file mode 100644 index 000000000..6854701f9 --- /dev/null +++ b/g/core/types/gmap/string_int_map.go @@ -0,0 +1,132 @@ +package gmap + +import ( + "sync" +) + +type StringIntMap struct { + sync.RWMutex + m map[string]int +} + +func NewStringIntMap() *StringIntMap { + return &StringIntMap{ + m: make(map[string]int), + } +} + +// 哈希表克隆 +func (this *StringIntMap) Clone() *map[string]int { + m := make(map[string]int) + this.RLock() + for k, v := range this.m { + m[k] = v + } + this.RUnlock() + return &m +} + +// 设置键值对 +func (this *StringIntMap) Set(key string, val int) { + this.Lock() + this.m[key] = val + this.Unlock() +} + +// 批量设置键值对 +func (this *StringIntMap) BatchSet(m map[string]int) { + this.Lock() + for k, v := range m { + this.m[k] = v + } + this.Unlock() +} + +// 获取键值 +func (this *StringIntMap) Get(key string) (int) { + this.RLock() + val, _ := this.m[key] + this.RUnlock() + return val +} + +// 删除键值对 +func (this *StringIntMap) Remove(key string) { + this.Lock() + delete(this.m, key) + this.Unlock() +} + +// 批量删除键值对 +func (this *StringIntMap) BatchRemove(keys []string) { + this.Lock() + for _, key := range keys { + delete(this.m, key) + } + this.Unlock() +} + +// 返回对应的键值，并删除该键值 +func (this *StringIntMap) GetAndRemove(key string) (int) { + this.Lock() + val, exists := this.m[key] + if exists { + delete(this.m, key) + } + this.Unlock() + return val +} + +// 返回键列表 +func (this *StringIntMap) Keys() []string { + this.RLock() + keys := make([]string, 0) + for key, _ := range this.m { + keys = append(keys, key) + } + this.RUnlock() + return keys +} + +// 返回值列表(注意是随机排序) +func (this *StringIntMap) Values() []int { + this.RLock() + vals := make([]int, 0) + for _, val := range this.m { + vals = append(vals, val) + } + this.RUnlock() + return vals +} + +// 是否存在某个键 +func (this *StringIntMap) Contains(key string) bool { + this.RLock() + _, exists := this.m[key] + this.RUnlock() + return exists +} + +// 哈希表大小 +func (this *StringIntMap) Size() int { + this.RLock() + len := len(this.m) + this.RUnlock() + return len +} + +// 哈希表是否为空 +func (this *StringIntMap) IsEmpty() bool { + this.RLock() + empty := (len(this.m) == 0) + this.RUnlock() + return empty +} + +// 清空哈希表 +func (this *StringIntMap) Clear() { + this.Lock() + this.m = make(map[string]int) + this.Unlock() +} + diff --git a/g/core/types/gmap/string_interface_map.go b/g/core/types/gmap/string_interface_map.go new file mode 100644 index 000000000..194d0c508 --- /dev/null +++ b/g/core/types/gmap/string_interface_map.go @@ -0,0 +1,131 @@ +package gmap + +import ( + "sync" +) + +type StringInterfaceMap struct { + sync.RWMutex + m map[string]interface{} +} + +func NewStringInterfaceMap() *StringInterfaceMap { + return &StringInterfaceMap{ + m: make(map[string]interface{}), + } +} + +// 哈希表克隆 +func (this *StringInterfaceMap) Clone() *map[string]interface{} { + m := make(map[string]interface{}) + this.RLock() + for k, v := range this.m { + m[k] = v + } + this.RUnlock() + return &m +} + +// 设置键值对 +func (this *StringInterfaceMap) Set(key string, val interface{}) { + this.Lock() + this.m[key] = val + this.Unlock() +} + +// 批量设置键值对 +func (this *StringInterfaceMap) BatchSet(m map[string]interface{}) { + this.Lock() + for k, v := range m { + this.m[k] = v + } + this.Unlock() +} + +// 获取键值 +func (this *StringInterfaceMap) Get(key string) interface{} { + this.RLock() + val, _ := this.m[key] + this.RUnlock() + return val +} + +// 删除键值对 +func (this *StringInterfaceMap) Remove(key string) { + this.Lock() + delete(this.m, key) + this.Unlock() +} + +// 批量删除键值对 +func (this *StringInterfaceMap) BatchRemove(keys []string) { + this.Lock() + for _, key := range keys { + delete(this.m, key) + } + this.Unlock() +} + +// 返回对应的键值，并删除该键值 +func (this *StringInterfaceMap) GetAndRemove(key string) interface{} { + this.Lock() + val, exists := this.m[key] + if exists { + delete(this.m, key) + } + this.Unlock() + return val +} + +// 返回键列表 +func (this *StringInterfaceMap) Keys() []string { + this.RLock() + keys := make([]string, 0) + for key, _ := range this.m { + keys = append(keys, key) + } + this.RUnlock() + return keys +} + +// 返回值列表(注意是随机排序) +func (this *StringInterfaceMap) Values() []interface{} { + this.RLock() + vals := make([]interface{}, 0) + for _, val := range this.m { + vals = append(vals, val) + } + this.RUnlock() + return vals +} + +// 是否存在某个键 +func (this *StringInterfaceMap) Contains(key string) bool { + this.RLock() + _, exists := this.m[key] + this.RUnlock() + return exists +} + +// 哈希表大小 +func (this *StringInterfaceMap) Size() int { + this.RLock() + len := len(this.m) + this.RUnlock() + return len +} + +// 哈希表是否为空 +func (this *StringInterfaceMap) IsEmpty() bool { + this.RLock() + empty := (len(this.m) == 0) + this.RUnlock() + return empty +} + +// 清空哈希表 +func (this *StringInterfaceMap) Clear() { + this.Lock() + this.m = make(map[string]interface{}) + this.Unlock() +} diff --git a/g/core/types/gmap/string_string_map.go b/g/core/types/gmap/string_string_map.go new file mode 100644 index 000000000..5db0ceff4 --- /dev/null +++ b/g/core/types/gmap/string_string_map.go @@ -0,0 +1,131 @@ +package gmap + +import ( + "sync" +) + +type StringStringMap struct { + sync.RWMutex + m map[string]string +} + +func NewStringStringMap() *StringStringMap { + return &StringStringMap{ + m: make(map[string]string), + } +} + +// 哈希表克隆 +func (this *StringStringMap) Clone() *map[string]string { + m := make(map[string]string) + this.RLock() + for k, v := range this.m { + m[k] = v + } + this.RUnlock() + return &m +} + +// 设置键值对 +func (this *StringStringMap) Set(key string, val string) { + this.Lock() + this.m[key] = val + this.Unlock() +} + +// 批量设置键值对 +func (this *StringStringMap) BatchSet(m map[string]string) { + this.Lock() + for k, v := range m { + this.m[k] = v + } + this.Unlock() +} + +// 获取键值 +func (this *StringStringMap) Get(key string) string { + this.RLock() + val, _ := this.m[key] + this.RUnlock() + return val +} + +// 删除键值对 +func (this *StringStringMap) Remove(key string) { + this.Lock() + delete(this.m, key) + this.Unlock() +} + +// 批量删除键值对 +func (this *StringStringMap) BatchRemove(keys []string) { + this.Lock() + for _, key := range keys { + delete(this.m, key) + } + this.Unlock() +} + +// 返回对应的键值，并删除该键值 +func (this *StringStringMap) GetAndRemove(key string) string { + this.Lock() + val, exists := this.m[key] + if exists { + delete(this.m, key) + } + this.Unlock() + return val +} + +// 返回键列表 +func (this *StringStringMap) Keys() []string { + this.RLock() + keys := make([]string, 0) + for key, _ := range this.m { + keys = append(keys, key) + } + this.RUnlock() + return keys +} + +// 返回值列表(注意是随机排序) +func (this *StringStringMap) Values() []string { + this.RLock() + vals := make([]string, 0) + for _, val := range this.m { + vals = append(vals, val) + } + this.RUnlock() + return vals +} + +// 是否存在某个键 +func (this *StringStringMap) Contains(key string) bool { + this.RLock() + _, exists := this.m[key] + this.RUnlock() + return exists +} + +// 哈希表大小 +func (this *StringStringMap) Size() int { + this.RLock() + len := len(this.m) + this.RUnlock() + return len +} + +// 哈希表是否为空 +func (this *StringStringMap) IsEmpty() bool { + this.RLock() + empty := (len(this.m) == 0) + this.RUnlock() + return empty +} + +// 清空哈希表 +func (this *StringStringMap) Clear() { + this.Lock() + this.m = make(map[string]string) + this.Unlock() +} diff --git a/g/core/types/gmap/uint_interface_map.go b/g/core/types/gmap/uint_interface_map.go new file mode 100644 index 000000000..df39ed189 --- /dev/null +++ b/g/core/types/gmap/uint_interface_map.go @@ -0,0 +1,132 @@ +package gmap + +import ( + "sync" +) + +type UintInterfaceMap struct { + sync.RWMutex + m map[uint]interface{} +} + +func NewUintInterfaceMap() *UintInterfaceMap { + return &UintInterfaceMap{ + m: make(map[uint]interface{}), + } +} + +// 哈希表克隆 +func (this *UintInterfaceMap) Clone() *map[uint]interface{} { + m := make(map[uint]interface{}) + this.RLock() + for k, v := range this.m { + m[k] = v + } + this.RUnlock() + return &m +} + +// 设置键值对 +func (this *UintInterfaceMap) Set(key uint, val interface{}) { + this.Lock() + this.m[key] = val + this.Unlock() +} + +// 批量设置键值对 +func (this *UintInterfaceMap) BatchSet(m map[uint]interface{}) { + this.Lock() + for k, v := range m { + this.m[k] = v + } + this.Unlock() +} + +// 获取键值 +func (this *UintInterfaceMap) Get(key uint) (interface{}) { + this.RLock() + val, _ := this.m[key] + this.RUnlock() + return val +} + +// 删除键值对 +func (this *UintInterfaceMap) Remove(key uint) { + this.Lock() + delete(this.m, key) + this.Unlock() +} + +// 批量删除键值对 +func (this *UintInterfaceMap) BatchRemove(keys []uint) { + this.Lock() + for _, key := range keys { + delete(this.m, key) + } + this.Unlock() +} + +// 返回对应的键值，并删除该键值 +func (this *UintInterfaceMap) GetAndRemove(key uint) (interface{}) { + this.Lock() + val, exists := this.m[key] + if exists { + delete(this.m, key) + } + this.Unlock() + return val +} + +// 返回键列表 +func (this *UintInterfaceMap) Keys() []uint { + this.RLock() + keys := make([]uint, 0) + for key, _ := range this.m { + keys = append(keys, key) + } + this.RUnlock() + return keys +} + +// 返回值列表(注意是随机排序) +func (this *UintInterfaceMap) Values() []interface{} { + this.RLock() + vals := make([]interface{}, 0) + for _, val := range this.m { + vals = append(vals, val) + } + this.RUnlock() + return vals +} + +// 是否存在某个键 +func (this *UintInterfaceMap) Contains(key uint) bool { + this.RLock() + _, exists := this.m[key] + this.RUnlock() + return exists +} + +// 哈希表大小 +func (this *UintInterfaceMap) Size() int { + this.RLock() + len := len(this.m) + this.RUnlock() + return len +} + +// 哈希表是否为空 +func (this *UintInterfaceMap) IsEmpty() bool { + this.RLock() + empty := (len(this.m) == 0) + this.RUnlock() + return empty +} + +// 清空哈希表 +func (this *UintInterfaceMap) Clear() { + this.Lock() + this.m = make(map[uint]interface{}) + this.Unlock() +} + diff --git a/g/core/types/gset/int64_set.go b/g/core/types/gset/int64_set.go new file mode 100644 index 000000000..a828b6ff2 --- /dev/null +++ b/g/core/types/gset/int64_set.go @@ -0,0 +1,108 @@ +package gset + +import ( + "fmt" + "sync" +) + +type Int64Set struct { + sync.RWMutex + M map[int64]struct{} +} + +func NewInt64Set() *Int64Set { + return &Int64Set{M: make(map[int64]struct{})} +} + +// 设置键 +func (this *Int64Set) Add(item int64) *Int64Set { + if this.Contains(item) { + return this + } + this.Lock() + this.M[item] = struct{}{} + this.Unlock() + return this +} + +// 批量添加设置键 +func (this *Int64Set) BatchAdd(items []int64) *Int64Set { + count := len(items) + if count == 0 { + return this + } + + todo := make([]int64, 0, count) + this.RLock() + for i := 0; i < count; i++ { + _, exists := this.M[items[i]] + if exists { + continue + } + + todo = append(todo, items[i]) + } + this.RUnlock() + + count = len(todo) + if count == 0 { + return this + } + + this.Lock() + for i := 0; i < count; i++ { + this.M[todo[i]] = struct{}{} + } + this.Unlock() + return this +} + +// 键是否存在 +func (this *Int64Set) Contains(item int64) bool { + this.RLock() + _, exists := this.M[item] + this.RUnlock() + return exists +} + +// 删除键值对 +func (this *Int64Set) Remove(key int64) { + this.Lock() + delete(this.M, key) + this.Unlock() +} + +// 大小 +func (this *Int64Set) Size() int { + this.RLock() + l := len(this.M) + this.RUnlock() + return l +} + +// 清空set +func (this *Int64Set) Clear() { + this.Lock() + this.M = make(map[int64]struct{}) + this.Unlock() +} + +// 转换为数组 +func (this *Int64Set) Slice() []int64 { + this.RLock() + ret := make([]int64, len(this.M)) + i := 0 + for item := range this.M { + ret[i] = item + i++ + } + + this.RUnlock() + return ret +} + +// 转换为字符串 +func (this *Int64Set) String() string { + s := this.Slice() + return fmt.Sprint(s) +} diff --git a/g/core/types/gset/int_set.go b/g/core/types/gset/int_set.go new file mode 100644 index 000000000..643c26167 --- /dev/null +++ b/g/core/types/gset/int_set.go @@ -0,0 +1,108 @@ +package gset + +import ( + "fmt" + "sync" +) + +type IntSet struct { + sync.RWMutex + M map[int]struct{} +} + +func NewIntSet() *IntSet { + return &IntSet{M: make(map[int]struct{})} +} + +// 设置键 +func (this *IntSet) Add(item int) *IntSet { + if this.Contains(item) { + return this + } + this.Lock() + this.M[item] = struct{}{} + this.Unlock() + return this +} + +// 批量添加设置键 +func (this *IntSet) BatchAdd(items []int) *IntSet { + count := len(items) + if count == 0 { + return this + } + + todo := make([]int, 0, count) + this.RLock() + for i := 0; i < count; i++ { + _, exists := this.M[items[i]] + if exists { + continue + } + + todo = append(todo, items[i]) + } + this.RUnlock() + + count = len(todo) + if count == 0 { + return this + } + + this.Lock() + for i := 0; i < count; i++ { + this.M[todo[i]] = struct{}{} + } + this.Unlock() + return this +} + +// 键是否存在 +func (this *IntSet) Contains(item int) bool { + this.RLock() + _, exists := this.M[item] + this.RUnlock() + return exists +} + +// 删除键值对 +func (this *IntSet) Remove(key int) { + this.Lock() + delete(this.M, key) + this.Unlock() +} + +// 大小 +func (this *IntSet) Size() int { + this.RLock() + l := len(this.M) + this.RUnlock() + return l +} + +// 清空set +func (this *IntSet) Clear() { + this.Lock() + this.M = make(map[int]struct{}) + this.Unlock() +} + +// 转换为数组 +func (this *IntSet) Slice() []int { + this.RLock() + ret := make([]int, len(this.M)) + i := 0 + for item := range this.M { + ret[i] = item + i++ + } + + this.RUnlock() + return ret +} + +// 转换为字符串 +func (this *IntSet) String() string { + s := this.Slice() + return fmt.Sprint(s) +} diff --git a/g/core/types/gset/interface_set.go b/g/core/types/gset/interface_set.go new file mode 100644 index 000000000..d567d6625 --- /dev/null +++ b/g/core/types/gset/interface_set.go @@ -0,0 +1,108 @@ +package gset + +import ( + "fmt" + "sync" +) + +type InterfaceSet struct { + sync.RWMutex + M map[interface{}]struct{} +} + +func NewInterfaceSet() *InterfaceSet { + return &InterfaceSet{M: make(map[interface{}]struct{})} +} + +// 设置键 +func (this *InterfaceSet) Add(item interface{}) *InterfaceSet { + if this.Contains(item) { + return this + } + this.Lock() + this.M[item] = struct{}{} + this.Unlock() + return this +} + +// 批量添加设置键 +func (this *InterfaceSet) BatchAdd(items []interface{}) *InterfaceSet { + count := len(items) + if count == 0 { + return this + } + + todo := make([]interface{}, 0, count) + this.RLock() + for i := 0; i < count; i++ { + _, exists := this.M[items[i]] + if exists { + continue + } + + todo = append(todo, items[i]) + } + this.RUnlock() + + count = len(todo) + if count == 0 { + return this + } + + this.Lock() + for i := 0; i < count; i++ { + this.M[todo[i]] = struct{}{} + } + this.Unlock() + return this +} + +// 键是否存在 +func (this *InterfaceSet) Contains(item interface{}) bool { + this.RLock() + _, exists := this.M[item] + this.RUnlock() + return exists +} + +// 删除键值对 +func (this *InterfaceSet) Remove(key interface{}) { + this.Lock() + delete(this.M, key) + this.Unlock() +} + +// 大小 +func (this *InterfaceSet) Size() int { + this.RLock() + l := len(this.M) + this.RUnlock() + return l +} + +// 清空set +func (this *InterfaceSet) Clear() { + this.Lock() + this.M = make(map[interface{}]struct{}) + this.Unlock() +} + +// 转换为数组 +func (this *InterfaceSet) Slice() []interface{} { + this.RLock() + ret := make([]interface{}, len(this.M)) + i := 0 + for item := range this.M { + ret[i] = item + i++ + } + + this.RUnlock() + return ret +} + +// 转换为字符串 +func (this *InterfaceSet) String() string { + s := this.Slice() + return fmt.Sprint(s) +} diff --git a/g/core/types/gset/string_set.go b/g/core/types/gset/string_set.go new file mode 100644 index 000000000..9e4780f06 --- /dev/null +++ b/g/core/types/gset/string_set.go @@ -0,0 +1,108 @@ +package gset + +import ( + "fmt" + "sync" +) + +type StringSet struct { + sync.RWMutex + M map[string]struct{} +} + +func NewStringSet() *StringSet { + return &StringSet{M: make(map[string]struct{})} +} + +// 设置键 +func (this *StringSet) Add(item string) *StringSet { + if this.Contains(item) { + return this + } + this.Lock() + this.M[item] = struct{}{} + this.Unlock() + return this +} + +// 批量添加设置键 +func (this *StringSet) BatchAdd(items []string) *StringSet { + count := len(items) + if count == 0 { + return this + } + + todo := make([]string, 0, count) + this.RLock() + for i := 0; i < count; i++ { + _, exists := this.M[items[i]] + if exists { + continue + } + + todo = append(todo, items[i]) + } + this.RUnlock() + + count = len(todo) + if count == 0 { + return this + } + + this.Lock() + for i := 0; i < count; i++ { + this.M[todo[i]] = struct{}{} + } + this.Unlock() + return this +} + +// 键是否存在 +func (this *StringSet) Contains(item string) bool { + this.RLock() + _, exists := this.M[item] + this.RUnlock() + return exists +} + +// 删除键值对 +func (this *StringSet) Remove(key string) { + this.Lock() + delete(this.M, key) + this.Unlock() +} + +// 大小 +func (this *StringSet) Size() int { + this.RLock() + l := len(this.M) + this.RUnlock() + return l +} + +// 清空set +func (this *StringSet) Clear() { + this.Lock() + this.M = make(map[string]struct{}) + this.Unlock() +} + +// 转换为数组 +func (this *StringSet) Slice() []string { + this.RLock() + ret := make([]string, len(this.M)) + i := 0 + for item := range this.M { + ret[i] = item + i++ + } + + this.RUnlock() + return ret +} + +// 转换为字符串 +func (this *StringSet) String() string { + s := this.Slice() + return fmt.Sprint(s) +} diff --git a/g/database/gdb/db.go b/g/database/gdb/db.go new file mode 100644 index 000000000..90d299140 --- /dev/null +++ b/g/database/gdb/db.go @@ -0,0 +1,348 @@ +// 对常用关系数据库的封装管理包 +package gdb + +import ( + "database/sql" + "errors" + "fmt" + "g/util/grand" + "sync" + "g/os/glog" + _ "github.com/go-sql-driver/mysql" + _ "github.com/lib/pq" + "g/os/gcache" +) + +const ( + OPTION_INSERT = 0 + OPTION_REPLACE = 1 + OPTION_SAVE = 2 + OPTION_IGNORE = 3 +) + +// 数据库配置包内对象 +var config struct { + sync.RWMutex + c Config // 数据库配置 + d string // 默认数据库分组名称 +} + +// 数据库操作接口 +type Link interface { + Open (c *ConfigNode) (*sql.DB, error) + Close() error + Query(q string, args ...interface{}) (*sql.Rows, error) + Exec(q string, args ...interface{}) (sql.Result, error) + Prepare(q string) (*sql.Stmt, error) + + GetAll(q string, args ...interface{}) (*List, error) + GetOne(q string, args ...interface{}) (*Map, error) + GetValue(q string, args ...interface{}) (interface{}, error) + + PingMaster() error + PingSlave() error + + SetMaxIdleConns(n int) + SetMaxOpenConns(n int) + + setMaster(master *sql.DB) + setSlave(slave *sql.DB) + setQuoteChar(left string, right string) + setLink(link Link) + getQuoteCharLeft () string + getQuoteCharRight () string + handleSqlBeforeExec(q *string) *string + + Begin() (*sql.Tx, error) + Commit() error + Rollback() error + + insert(table string, data *Map, option uint8) (sql.Result, error) + Insert(table string, data *Map) (sql.Result, error) + Replace(table string, data *Map) (sql.Result, error) + Save(table string, data *Map) (sql.Result, error) + + batchInsert(table string, list *List, batch int, option uint8) error + BatchInsert(table string, list *List, batch int) error + BatchReplace(table string, list *List, batch int) error + BatchSave(table string, list *List, batch int) error + + Update(table string, data interface{}, condition interface{}, args ...interface{}) (sql.Result, error) + Delete(table string, condition interface{}, args ...interface{}) (sql.Result, error) + + Table(tables string) (*gLinkOp) +} + +// 数据库链接对象 +type dbLink struct { + link Link + transaction *sql.Tx + master *sql.DB + slave *sql.DB + charl string + charr string +} + +// 数据库配置 +type Config map[string]ConfigGroup + +// 数据库集群配置 +type ConfigGroup []ConfigNode + +// 数据库单项配置 +type ConfigNode struct { + Host string // 地址 + Port string // 端口 + User string // 账号 + Pass string // 密码 + Name string // 数据库名称 + Type string // 数据库类型：mysql, sqlite, mssql, pgsql, oracle(目前仅支持mysql) + Role string // (可选，默认为master)数据库的角色，用于主从操作分离，至少需要有一个master，参数值：master, slave + Charset string // (可选，默认为 utf-8)编码，默认为 utf-8 + Priority int // (可选)用于负载均衡的权重计算，当集群中只有一个节点时，权重没有任何意义 + Linkinfo string // (可选)自定义链接信息，当该字段被设置值时，以上链接字段(Host,Port,User,Pass,Name)将失效(该字段是一个扩展功能) +} + +// 关联数组，绑定一条数据表记录 +type Map map[string]interface{} + +// 关联数组列表(索引从0开始的数组)，绑定多条记录 +type List []Map + +// 数据库集群配置示例，支持主从处理，多数据库集群支持 +/* +var DatabaseConfiguration = Config { + // 数据库集群配置名称 + "default" : ConfigGroup { + { + Host : "192.168.1.100", + Port : "3306", + User : "root", + Pass : "123456", + Name : "test", + Type : "mysql", + Role : "master", + Charset : "utf-8", + Priority : 100, + }, + { + Host : "192.168.1.101", + Port : "3306", + User : "root", + Pass : "123456", + Name : "test", + Type : "mysql", + Role : "slave", + Charset : "utf-8", + Priority : 100, + }, + }, +} +*/ + +// 包初始化 +func init() { + config.c = make(Config) + config.d = "default" +} + +// 设置当前应用的数据库配置信息，进行全局数据库配置覆盖操作 +// 支持三种数据类型的输入参数：Config, ConfigGroup, ConfigNode +func SetConfig (c interface{}) error { + config.Lock() + defer config.Unlock() + + switch c.(type) { + case Config: + config.c = c.(Config) + + case ConfigGroup: + config.c = Config {"default" : c.(ConfigGroup)} + + case ConfigNode: + config.c = Config {"default" : ConfigGroup { c.(ConfigNode) }} + + default: + return errors.New("invalid config type, types should be in: Config, ConfigGroup, ConfigNode") + } + return nil +} + +// 添加一台数据库服务器配置 +func AddConfigNode (group string, node ConfigNode) { + config.Lock() + config.c[group] = append(config.c[group], node) + config.Unlock() +} + +// 添加数据库服务器集群配置 +func AddConfigGroup (group string, nodes ConfigGroup) { + config.Lock() + config.c[group] = nodes + config.Unlock() +} + +// 添加默认链接的一台数据库服务器配置 +func AddDefaultConfigNode (node ConfigNode) { + AddConfigNode("default", node) +} + +// 添加默认链接的数据库服务器集群配置 +func AddDefaultConfigGroup (nodes ConfigGroup) { + AddConfigGroup("default", nodes) +} + +// 设置默认链接的数据库链接配置项(默认是 default) +func SetDefaultGroup (groupName string) { + config.Lock() + config.d = groupName + config.Unlock() +} + +// 根据配置项获取一个数据库操作对象单例 +func instance (groupName string) (Link, error) { + instanceName := "gdb_instance_" + groupName + result := gcache.Get(instanceName) + if result == nil { + link, err := NewByGroup(groupName) + if err == nil { + gcache.Set(instanceName, link, 0) + return link, nil + } else { + return nil, err + } + } else { + return result.(Link), nil + } +} + +// 获得默认的数据库操作对象单例 +func Instance () (Link, error) { + return instance(config.d) +} + +// 获得指定配置项的数据库草最对象单例 +func InstanceByGroup(groupName string) (Link, error) { + return instance(groupName) +} + +// 使用默认选项进行连接，数据库集群配置项：default +func New() (Link, error) { + return NewByGroup(config.d) +} + +// 根据数据库配置项创建一个数据库操作对象 +func NewByGroup(groupName string) (Link, error) { + config.RLock() + defer config.RUnlock() + + if len(config.c) < 1 { + return nil, errors.New("empty database configuration") + } + if list, ok := config.c[groupName]; ok { + // 将master, slave集群列表拆分出来 + masterList := make(ConfigGroup, 0) + slaveList := make(ConfigGroup, 0) + for i := 0; i < len(list); i++ { + if list[i].Role == "slave" { + slaveList = append(slaveList, list[i]) + } else { + // 默认配置项的角色为master + masterList = append(masterList, list[i]) + } + } + if len(masterList) < 1 { + return nil, errors.New("at least one master node configuration's need to make sense") + } + masterNode := getConfigNodeByPriority(&masterList) + var slaveNode *ConfigNode + if len(slaveList) > 0 { + slaveNode = getConfigNodeByPriority(&slaveList) + } + return newLink(masterNode, slaveNode) + } else { + return nil, errors.New(fmt.Sprintf("empty database configuration for item name '%s'", groupName)) + } +} + +// 根据单点数据库配置获得一个数据库草最对象 +func NewByConfigNode(node ConfigNode) (Link, error) { + return newLink (&node, nil) +} + +// 按照负载均衡算法(优先级配置)从数据库集群中选择一个配置节点出来使用 +func getConfigNodeByPriority (cg *ConfigGroup) *ConfigNode { + if len(*cg) < 2 { + return &(*cg)[0] + } + var total int + for i := 0; i < len(*cg); i++ { + total += (*cg)[i].Priority * 100 + } + r := grand.Rand(0, total) + min := 0 + max := 0 + for i := 0; i < len(*cg); i++ { + max = min + (*cg)[i].Priority * 100 + //fmt.Printf("r: %d, min: %d, max: %d\n", r, min, max) + if r >= min && r < max { + return &(*cg)[i] + } else { + min = max + } + } + return nil +} + +// 创建数据库链接对象 +func newLink (masterNode *ConfigNode, slaveNode *ConfigNode) (Link, error) { + var link Link + switch masterNode.Type { + case "mysql": + link = Link(&mysqlLink{}) + + case "pgsql": + link = Link(&pgsqlLink{}) + + default: + return nil, errors.New(fmt.Sprintf("unsupported db type '%s'", masterNode.Type)) + } + master, err := link.Open(masterNode) + if err != nil { + glog.Fatal(err) + } + slave := master + if slaveNode != nil { + slave, err = link.Open(slaveNode) + if err != nil { + glog.Fatal(err) + } + } + link.setLink(link) + link.setMaster(master) + link.setSlave(slave) + link.setQuoteChar(link.getQuoteCharLeft(), link.getQuoteCharRight()) + return link, nil +} + +// 设置master链接对象 +func (l *dbLink) setMaster(master *sql.DB) { + l.master = master +} + +// 设置slave链接对象 +func (l *dbLink) setSlave(slave *sql.DB) { + l.slave = slave +} + +// 设置当前数据库类型引用字符 +func (l *dbLink) setQuoteChar(left string, right string) { + l.charl = left + l.charr = right +} + +// 设置挡脸操作的link接口 +func (l *dbLink) setLink(link Link) { + l.link = link +} + diff --git a/g/database/gdb/db_base.go b/g/database/gdb/db_base.go new file mode 100644 index 000000000..0e569af0a --- /dev/null +++ b/g/database/gdb/db_base.go @@ -0,0 +1,337 @@ +package gdb + +import ( + "fmt" + "errors" + "strings" + "database/sql" + "g/os/glog" +) + +// 关闭链接 +func (l *dbLink) Close() error { + if l.master != nil { + err := l.master.Close() + if (err == nil) { + l.master = nil + } else { + glog.Fatal(err) + return err + } + } + if l.slave != nil { + err := l.slave.Close() + if (err == nil) { + l.slave = nil + } else { + glog.Fatal(err) + return err + } + } + return nil +} + +// 数据库sql查询操作，主要执行查询 +func (l *dbLink) Query(q string, args ...interface{}) (*sql.Rows, error) { + p := l.link.handleSqlBeforeExec(&q) + rows, err := l.slave.Query(*p, args ...) + err = l.formatError(err, p, args...) + if (err == nil) { + return rows, nil + } + return nil, err +} + +// 执行一条sql，并返回执行情况，主要用于非查询操作 +func (l *dbLink) Exec(q string, args ...interface{}) (sql.Result, error) { + //fmt.Println(q) + //fmt.Println(args) + p := l.link.handleSqlBeforeExec(&q) + r, err := l.master.Exec(*p, args ...) + err = l.formatError(err, p, args...) + return r, err +} + +// 格式化错误信息 +func (l *dbLink) formatError(err error, q *string, args ...interface{}) error { + if err != nil { + errstr := fmt.Sprintf("DB ERROR: %s\n", err.Error()) + errstr += fmt.Sprintf("DB QUERY: %s\n", *q) + if len(args) > 0 { + errstr += fmt.Sprintf("DB PARAM: %v\n", args) + } + err = errors.New(errstr) + } + return err +} + + +// 数据库查询，获取查询结果集，以列表结构返回 +func (l *dbLink) GetAll(q string, args ...interface{}) (*List, error) { + // 执行sql + rows, err := l.Query(q, args ...) + if err != nil || rows == nil { + return nil, err + } + // 列名称列表 + columns, err := rows.Columns() + if err != nil { + return nil, err + } + // 返回结构组装 + values := make([]sql.RawBytes, len(columns)) + scanArgs := make([]interface{}, len(values)) + var list List + for i := range values { + scanArgs[i] = &values[i] + } + for rows.Next() { + err = rows.Scan(scanArgs...) + if err != nil { + return &list, err + } + row := make(Map) + for i, col := range values { + row[columns[i]] = string(col) + } + list = append(list, row) + } + return &list, nil +} + +// 数据库查询，获取查询结果集，以关联数组结构返回 +func (l *dbLink) GetOne(q string, args ...interface{}) (*Map, error) { + list, err := l.GetAll(q, args ...) + if err != nil { + return nil, err + } + return &(*list)[0], nil +} + +// 数据库查询，获取查询字段值 +func (l *dbLink) GetValue(q string, args ...interface{}) (interface{}, error) { + one, err := l.GetOne(q, args ...) + if err != nil { + return "", err + } + for _, v := range *one { + return v, nil + } + return "", nil +} + +// sql预处理，执行完成后调用返回值sql.Stmt.Exec完成sql操作 +// 记得调用sql.Stmt.Close关闭操作对象 +func (l *dbLink) Prepare(q string) (*sql.Stmt, error) { + return l.master.Prepare(q) +} + +// ping一下，判断或保持数据库链接(master) +func (l *dbLink) PingMaster() error { + err := l.master.Ping(); + return err +} + +// ping一下，判断或保持数据库链接(slave) +func (l *dbLink) PingSlave() error { + err := l.slave.Ping(); + return err +} + +// 设置数据库连接池中空闲链接的大小 +func (l *dbLink) SetMaxIdleConns(n int) { + l.master.SetMaxIdleConns(n); +} + +// 设置数据库连接池最大打开的链接数量 +func (l *dbLink) SetMaxOpenConns(n int) { + l.master.SetMaxOpenConns(n); +} + +// 事务操作，开启，会返回一个底层的事务操作对象链接如需要嵌套事务，那么可以使用该对象，否则请忽略 +func (l *dbLink) Begin() (*sql.Tx, error) { + tx, err := l.master.Begin() + if err == nil { + l.transaction = tx + } + return tx, err +} + +// 事务操作，提交 +func (l *dbLink) Commit() error { + if l.transaction == nil { + return errors.New("transaction not start") + } + err := l.transaction.Commit() + return err +} + +// 事务操作，回滚 +func (l *dbLink) Rollback() error { + if l.transaction == nil { + return errors.New("transaction not start") + } + err := l.transaction.Rollback() + return err +} + +// 根据insert选项获得操作名称 +func (l *dbLink) getInsertOperationByOption(option uint8) string { + oper := "INSERT" + switch option { + case OPTION_INSERT: + case OPTION_REPLACE: + oper = "REPLACE" + case OPTION_SAVE: + case OPTION_IGNORE: + oper = "INSERT IGNORE" + } + return oper +} + +// insert、replace, save， ignore操作 +// 0: insert: 仅仅执行写入操作，如果存在冲突的主键或者唯一索引，那么报错返回 +// 1: replace: 如果数据存在(主键或者唯一索引)，那么删除后重新写入一条 +// 2: save: 如果数据存在(主键或者唯一索引)，那么更新，否则写入一条新数据 +// 3: ignore: 如果数据存在(主键或者唯一索引)，那么什么也不做 +func (l *dbLink) insert(table string, data *Map, option uint8) (sql.Result, error) { + var keys []string + var values []string + var params []interface{} + for k, v := range *data { + keys = append(keys, l.charl + k + l.charr) + values = append(values, "?") + params = append(params, v) + } + operation := l.getInsertOperationByOption(option) + updatestr := "" + if option == OPTION_SAVE { + var updates []string + for k, _ := range *data { + updates = append(updates, fmt.Sprintf("%s%s%s=VALUES(%s)", l.charl, k, l.charr, k)) + } + updatestr = fmt.Sprintf(" ON DUPLICATE KEY UPDATE %s", strings.Join(updates, ",")) + } + return l.Exec( + fmt.Sprintf("%s INTO %s%s%s(%s) VALUES(%s) %s", + operation, l.charl, table, l.charr, strings.Join(keys, ","), strings.Join(values, ","), updatestr), params... + ) +} + +// CURD操作:单条数据写入, 仅仅执行写入操作，如果存在冲突的主键或者唯一索引，那么报错返回 +func (l *dbLink) Insert(table string, data *Map) (sql.Result, error) { + return l.link.insert(table, data, OPTION_INSERT) +} + +// CURD操作:单条数据写入, 如果数据存在(主键或者唯一索引)，那么删除后重新写入一条 +func (l *dbLink) Replace(table string, data *Map) (sql.Result, error) { + return l.link.insert(table, data, OPTION_REPLACE) +} + +// CURD操作:单条数据写入, 如果数据存在(主键或者唯一索引)，那么更新，否则写入一条新数据 +func (l *dbLink) Save(table string, data *Map) (sql.Result, error) { + return l.link.insert(table, data, OPTION_SAVE) +} + +// 批量写入数据 +func (l *dbLink) batchInsert(table string, list *List, batch int, option uint8) error { + var keys []string + var values []string + var bvalues []string + var params []interface{} + var size int = len(*list) + // 判断长度 + if size < 1 { + return errors.New("empty data list") + } + // 首先获取字段名称及记录长度 + for k, _ := range (*list)[0] { + keys = append(keys, k) + values = append(values, "?") + } + var kstr = l.charl + strings.Join(keys, l.charl + "," + l.charr) + l.charr + // 操作判断 + operation := l.getInsertOperationByOption(option) + updatestr := "" + if option == OPTION_SAVE { + var updates []string + for _, k := range keys { + updates = append(updates, fmt.Sprintf("%s=VALUES(%s)", l.charl, k, l.charr, k)) + } + updatestr = fmt.Sprintf(" ON DUPLICATE KEY UPDATE %s", strings.Join(updates, ",")) + } + // 构造批量写入数据格式(注意map的遍历是无序的) + for i := 0; i < size; i++ { + for _, k := range keys { + params = append(params, (*list)[i][k]) + } + bvalues = append(bvalues, "(" + strings.Join(values, ",") + ")") + if len(bvalues) == batch { + _, err := l.Exec(fmt.Sprintf("%s INTO %s%s%s(%s) VALUES%s %s", operation, l.charl, table, l.charr, kstr, strings.Join(bvalues, ","), updatestr), params...) + if err != nil { + return err + } + bvalues = bvalues[:0] + } + } + // 处理最后不构成指定批量的数据 + if (len(bvalues) > 0) { + _, err := l.Exec(fmt.Sprintf("%s INTO %s%s%s(%s) VALUES%s %s", operation, l.charl, table, l.charr, kstr, strings.Join(bvalues, ","), updatestr), params...) + if err != nil { + return err + } + } + return nil +} + +// CURD操作:批量数据指定批次量写入 +func (l *dbLink) BatchInsert(table string, list *List, batch int) error { + return l.link.batchInsert(table, list, batch, OPTION_INSERT) +} + +// CURD操作:批量数据指定批次量写入, 如果数据存在(主键或者唯一索引)，那么删除后重新写入一条 +func (l *dbLink) BatchReplace(table string, list *List, batch int) error { + return l.link.batchInsert(table, list, batch, OPTION_REPLACE) +} + +// CURD操作:批量数据指定批次量写入, 如果数据存在(主键或者唯一索引)，那么更新，否则写入一条新数据 +func (l *dbLink) BatchSave(table string, list *List, batch int) error { + return l.link.batchInsert(table, list, batch, OPTION_SAVE) +} + +// CURD操作:数据更新，统一采用sql预处理 +// data参数支持字符串或者关联数组类型，内部会自行做判断处理 +func (l *dbLink) Update(table string, data interface{}, condition interface{}, args ...interface{}) (sql.Result, error) { + var params []interface{} + var updates string + switch data.(type) { + case string: + updates = data.(string) + case *Map: + var keys []string + for k, v := range *data.(*Map) { + keys = append(keys, fmt.Sprintf("%s%s%s=?", l.charl, k, l.charr)) + params = append(params, v) + } + updates = strings.Join(keys, ",") + + default: + return nil, errors.New("invalid data type for 'data' field, string or *Map expected") + } + for _, v := range args { + if r, ok := v.(string); ok { + params = append(params, r) + } else if r, ok := v.(int); ok { + params = append(params, string(r)) + } else { + + } + } + return l.Exec(fmt.Sprintf("UPDATE %s%s%s SET %s WHERE %s", l.charl, table, l.charr, updates, condition), params...) +} + +// CURD操作:删除数据 +func (l *dbLink) Delete(table string, condition interface{}, args ...interface{}) (sql.Result, error) { + return l.Exec(fmt.Sprintf("DELETE FROM %s WHERE %s", l.charl, table, l.charr, condition), args...) +} + diff --git a/g/database/gdb/db_linkop.go b/g/database/gdb/db_linkop.go new file mode 100644 index 000000000..1f99765ea --- /dev/null +++ b/g/database/gdb/db_linkop.go @@ -0,0 +1,236 @@ +package gdb + +import ( + "fmt" + "database/sql" + "errors" + _ "github.com/go-sql-driver/mysql" +) + +// gf的数据库操作支持普通方法操作及链式操作两种方式，本文件是链式操作的封装，提供非常简便的CURD方法 + + +// 数据库链式操作对象 +type gLinkOp struct { + link Link + tables string + fields string + condition string + conditionArgs []interface{} + groupby string + orderby string + start int + limit int + data interface{} + dataList *List + batch int +} + +// 链式操作，数据表字段，可支持多个表，以半角逗号连接 +func (l *dbLink) Table(tables string) (*gLinkOp) { + return &gLinkOp{ + link : l.link, + tables: tables, + } +} + +// 链式操作，左联表 +func (op *gLinkOp) LeftJoin(joinTable string, on string) (*gLinkOp) { + op.tables += fmt.Sprintf(" LEFT JOIN %s ON (%s)", joinTable, on) + return op +} + +// 链式操作，右联表 +func (op *gLinkOp) RightJoin(joinTable string, on string) (*gLinkOp) { + op.tables += fmt.Sprintf(" RIGHT JOIN %s ON (%s)", joinTable, on) + return op +} + +// 链式操作，内联表 +func (op *gLinkOp) InnerJoin(joinTable string, on string) (*gLinkOp) { + op.tables += fmt.Sprintf(" INNER JOIN %s ON (%s)", joinTable, on) + return op +} + +// 链式操作，查询字段 +func (op *gLinkOp) Fields(fields string) (*gLinkOp) { + op.fields = fields + return op +} + +// 链式操作，consition +func (op *gLinkOp) Condition(condition string, args...interface{}) (*gLinkOp) { + op.condition = condition + op.conditionArgs = args + return op +} + +// 链式操作，group by +func (op *gLinkOp) GroupBy(groupby string) (*gLinkOp) { + op.groupby = groupby + return op +} + +// 链式操作，order by +func (op *gLinkOp) OrderBy(orderby string) (*gLinkOp) { + op.orderby = orderby + return op +} + +// 链式操作，limit +func (op *gLinkOp) Limit(start int, limit int) (*gLinkOp) { + op.start = start + op.limit = limit + return op +} + +// 链式操作，操作数据记录项 +func (op *gLinkOp) Data(data interface{}) (*gLinkOp) { + op.data = data + return op +} + +// 链式操作，操作数据记录项列表 +func (op *gLinkOp) List(list *List) (*gLinkOp) { + op.dataList = list + return op +} + +// 链式操作， CURD - Insert +func (op *gLinkOp) Insert() (sql.Result, error) { + if op.data == nil { + return nil, errors.New("inserting into table with empty data") + } + if d, ok := op.data.(*Map); ok { + return op.link.Insert(op.tables, d) + } + return nil, errors.New("inserting into table with invalid data type") +} + +// 链式操作， CURD - Replace +func (op *gLinkOp) Replace() (sql.Result, error) { + if op.data == nil { + return nil, errors.New("replacing into table with empty data") + } + if d, ok := op.data.(*Map); ok { + return op.link.Insert(op.tables, d) + } + return nil, errors.New("replacing into table with invalid data type") +} + +// 链式操作， CURD - Save +func (op *gLinkOp) Save() (sql.Result, error) { + if op.data == nil { + return nil, errors.New("saving into table with empty data") + } + if d, ok := op.data.(*Map); ok { + return op.link.Insert(op.tables, d) + } + return nil, errors.New("saving into table with invalid data type") +} + +// 设置批处理的大小 +func (op *gLinkOp) Batch(batch int) *gLinkOp { + op.batch = batch + return op +} + +// 链式操作， CURD - BatchInsert +func (op *gLinkOp) BatchInsert() error { + if op.dataList == nil || len(*op.dataList) < 1 { + return errors.New("batch inserting into table with empty data list") + } + batch := 10 + if op.batch > 0 { + batch = op.batch + } + return op.link.BatchInsert(op.tables, op.dataList, batch) +} + +// 链式操作， CURD - BatchReplace +func (op *gLinkOp) BatchReplace() error { + if op.dataList == nil || len(*op.dataList) < 1 { + return errors.New("batch replacing into table with empty data list") + } + batch := 10 + if op.batch > 0 { + batch = op.batch + } + return op.link.BatchReplace(op.tables, op.dataList, batch) +} + +// 链式操作， CURD - BatchSave +func (op *gLinkOp) BatchSave() error { + if op.dataList == nil || len(*op.dataList) < 1 { + return errors.New("batch saving into table with empty data list") + } + batch := 10 + if op.batch > 0 { + batch = op.batch + } + return op.link.BatchSave(op.tables, op.dataList, batch) +} + +// 链式操作， CURD - Update +func (op *gLinkOp) Update() (sql.Result, error) { + if op.data == nil { + return nil, errors.New("updating table with empty data") + } + return op.link.Update(op.tables, op.data, op.condition, op.conditionArgs ...) +} + +// 链式操作， CURD - Delete +func (op *gLinkOp) Delete() (sql.Result, error) { + if op.condition == "" { + return nil, errors.New("condition is required while deleting") + } + return op.link.Delete(op.tables, op.condition, op.conditionArgs...) +} + +// 链式操作，select +func (op *gLinkOp) Select() (*List, error) { + if op.fields == "" { + op.fields = "*" + } + s := fmt.Sprintf("SELECT %s FROM %s", op.fields, op.tables) + if op.condition != "" { + s += " WHERE " + op.condition + } + if op.groupby != "" { + s += " GROUP BY " + op.groupby + } + if op.orderby != "" { + s += " ORDER BY " + op.orderby + } + if op.limit != 0 { + s += fmt.Sprintf(" LIMIT %d, %d", op.start, op.limit) + } + return op.link.GetAll(s, op.conditionArgs...) +} + +// 链式操作，查询所有记录 +func (op *gLinkOp) All() (*List, error) { + return op.Select() +} + +// 链式操作，查询单条记录 +func (op *gLinkOp) One() (*Map, error) { + list, err := op.All() + if err != nil { + return nil, err + } + return &(*list)[0], nil +} + +// 链式操作，查询字段值 +func (op *gLinkOp) Value() (interface{}, error) { + one, err := op.One() + if err != nil { + return "", err + } + for _, v := range *one { + return v, nil + } + return "", nil +} + diff --git a/g/database/gdb/db_mysql.go b/g/database/gdb/db_mysql.go new file mode 100644 index 000000000..45673c588 --- /dev/null +++ b/g/database/gdb/db_mysql.go @@ -0,0 +1,42 @@ +package gdb + +import ( + "database/sql" + "fmt" + "g/os/glog" +) + +// 数据库链接对象 +type mysqlLink struct { + dbLink +} + +// 创建SQL操作对象，内部采用了lazy link处理 +func (l *mysqlLink) Open (c *ConfigNode) (*sql.DB, error) { + var dbsource string + if c.Linkinfo != "" { + dbsource = c.Linkinfo + } else { + dbsource = fmt.Sprintf("%s:%s@tcp(%s:%s)/%s", c.User, c.Pass, c.Host, c.Port, c.Name) + } + db, err := sql.Open("mysql", dbsource) + if err != nil { + glog.Fatal(err) + } + return db, err +} + +// 获得关键字操作符 - 左 +func (l *mysqlLink) getQuoteCharLeft () string { + return "`" +} + +// 获得关键字操作符 - 右 +func (l *mysqlLink) getQuoteCharRight () string { + return "`" +} + +// 在执行sql之前对sql进行进一步处理 +func (l *mysqlLink) handleSqlBeforeExec(q *string) *string { + return q +} \ No newline at end of file diff --git a/g/database/gdb/db_pgsql.go b/g/database/gdb/db_pgsql.go new file mode 100644 index 000000000..a2e512674 --- /dev/null +++ b/g/database/gdb/db_pgsql.go @@ -0,0 +1,53 @@ +package gdb + +import ( + "database/sql" + "fmt" + "regexp" + "g/os/glog" +) + +// postgresql的适配 +// @todo 需要完善replace和save的操作覆盖 + +// 数据库链接对象 +type pgsqlLink struct { + dbLink +} + +// 创建SQL操作对象，内部采用了lazy link处理 +func (l *pgsqlLink) Open (c *ConfigNode) (*sql.DB, error) { + var dbsource string + if c.Linkinfo != "" { + dbsource = c.Linkinfo + } else { + dbsource = fmt.Sprintf("user=%s password=%s host=%s port=%s dbname=%s", c.User, c.Pass, c.Host, c.Port, c.Name) + } + db, err := sql.Open("postgres", dbsource) + if err != nil { + glog.Fatal(err) + } + return db, err +} + +// 获得关键字操作符 - 左 +func (l *pgsqlLink) getQuoteCharLeft () string { + return "\"" +} + +// 获得关键字操作符 - 右 +func (l *pgsqlLink) getQuoteCharRight () string { + return "\"" +} + +// 在执行sql之前对sql进行进一步处理 +func (l *pgsqlLink) handleSqlBeforeExec(q *string) *string { + reg := regexp.MustCompile("\\?") + index := 0 + str := reg.ReplaceAllStringFunc(*q, func (s string) string { + index ++ + return fmt.Sprintf("$%d", index) + }) + return &str +} + diff --git a/g/encoding/gbase64/gbase64.go b/g/encoding/gbase64/gbase64.go new file mode 100644 index 000000000..5e902515e --- /dev/null +++ b/g/encoding/gbase64/gbase64.go @@ -0,0 +1,16 @@ +package gbase64 + +import ( + "encoding/base64" +) + +// base64 encode +func Encode(str string) string { + return base64.StdEncoding.EncodeToString([]byte(str)) +} + +// base64 decode +func Decode(str string) (string, error) { + s, e := base64.StdEncoding.DecodeString(str) + return string(s), e +} \ No newline at end of file diff --git a/g/encoding/gbinary/gbinary.go b/g/encoding/gbinary/gbinary.go new file mode 100644 index 000000000..4a839147f --- /dev/null +++ b/g/encoding/gbinary/gbinary.go @@ -0,0 +1,197 @@ +// 二进制及byte操作管理包 +package gbinary + +import ( + "bytes" + "encoding/binary" + "math" +) + +// 二进制位(0|1) +type Bit uint8 + +// (通用,效率较低)二进制打包 +func Encode(vs ...interface{}) ([]byte, error) { + buf := new(bytes.Buffer) + for i := 0; i < len(vs); i++ { + err := binary.Write(buf, binary.LittleEndian, vs[i]) + if err != nil { + return nil, err + } + } + return buf.Bytes(), nil +} + +// (通用,效率较低)二进制解包，注意第二个参数之后的变量是变量的指针地址 +func Decode(b []byte, vs ...interface{}) error { + buf := bytes.NewBuffer(b) + for i := 0; i < len(vs); i++ { + err := binary.Read(buf, binary.LittleEndian, vs[i]) + if err != nil { + return err + } + } + return nil +} + +func EncodeString(s string) []byte { + return []byte(s) +} + +func DecodeToString(b []byte) string { + return string(b) +} + +func EncodeInt8(i int8) []byte { + return []byte{byte(i)} +} + +func EncodeUint8(i uint8) []byte { + return []byte{byte(i)} +} + +func EncodeInt16(i int16) []byte { + bytes := make([]byte, 2) + binary.LittleEndian.PutUint16(bytes, uint16(i)) + return bytes +} + +func EncodeUint16(i uint16) []byte { + bytes := make([]byte, 2) + binary.LittleEndian.PutUint16(bytes, i) + return bytes +} + +func EncodeInt32(i int32) []byte { + bytes := make([]byte, 4) + binary.LittleEndian.PutUint32(bytes, uint32(i)) + return bytes +} + +func EncodeUint32(i uint32) []byte { + bytes := make([]byte, 4) + binary.LittleEndian.PutUint32(bytes, i) + return bytes +} + +func EncodeInt64(i int64) []byte { + bytes := make([]byte, 8) + binary.LittleEndian.PutUint64(bytes, uint64(i)) + return bytes +} + +func EncodeUint64(i uint64) []byte { + bytes := make([]byte, 8) + binary.LittleEndian.PutUint64(bytes, i) + return bytes +} + +func EncodeFloat32(f float32) []byte { + bits := math.Float32bits(f) + bytes := make([]byte, 4) + binary.LittleEndian.PutUint32(bytes, bits) + return bytes +} + +func EncodeFloat64(f float64) []byte { + bits := math.Float64bits(f) + bytes := make([]byte, 8) + binary.LittleEndian.PutUint64(bytes, bits) + return bytes +} + +// 当b位数不够时，进行高位补0 +func fillUpSize(b []byte, l int) []byte { + c := make([]byte, 0) + c = append(c, b...) + for i := 0; i <= l - len(b); i++ { + c = append(c, 0x00) + } + return c +} + +func DecodeToInt8(b []byte) int8 { + return int8(b[0]) +} + +func DecodeToUint8(b []byte) uint8 { + return uint8(b[0]) +} + +func DecodeToInt16(b []byte) int16 { + return int16(binary.LittleEndian.Uint16(fillUpSize(b, 2))) +} + +func DecodeToUint16(b []byte) uint16 { + return binary.LittleEndian.Uint16(fillUpSize(b, 2)) +} + +func DecodeToInt32(b []byte) int32 { + return int32(binary.LittleEndian.Uint32(fillUpSize(b, 4))) +} + +func DecodeToUint32(b []byte) uint32 { + return binary.LittleEndian.Uint32(fillUpSize(b, 4)) +} + +func DecodeToInt64(b []byte) int64 { + return int64(binary.LittleEndian.Uint64(fillUpSize(b, 8))) +} + +func DecodeToUint64(b []byte) uint64 { + return binary.LittleEndian.Uint64(fillUpSize(b, 8)) +} + +func DecodeToFloat32(b []byte) float32 { + return math.Float32frombits(binary.LittleEndian.Uint32(fillUpSize(b, 4))) +} + +func DecodeToFloat64(b []byte) float64 { + return math.Float64frombits(binary.LittleEndian.Uint64(fillUpSize(b, 8))) +} + +// 将ui按位合并到bits数组中，并占length长度位(注意：uis数组中存放的是二进制的0|1数字) +func EncodeBits(bits []Bit, ui uint, l int) []Bit { + a := make([]Bit, l) + for i := l - 1; i >= 0; i-- { + a[i] = Bit(ui & 1) + ui >>= 1 + } + if bits != nil { + return append(bits, a...) + } else { + return a + } +} + +// 将bits转换为[]byte，从左至右进行编码，不足1 byte按0往末尾补充 +func EncodeBitsToBytes(bits []Bit) []byte { + if len(bits)%8 != 0 { + for i := 0; i < len(bits)%8; i++ { + bits = append(bits, 0) + } + } + b := make([]byte, 0) + for i := 0; i < len(bits); i += 8 { + b = append(b, byte(DecodeBits(bits[i : i + 8]))) + } + return b +} + +// 从ui字位数组中解析为uint +func DecodeBits(bits []Bit) uint { + ui := uint(0) + for _, i := range bits { + ui = ui << 1 | uint(i) + } + return ui +} + +// 解析[]byte为字位数组[]uint8 +func DecodeBytesToBits(bs []byte) []Bit { + bits := make([]Bit, 0) + for _, b := range bs { + bits = EncodeBits(bits, uint(b), 8) + } + return bits +} \ No newline at end of file diff --git a/g/encoding/gcompress/gcompress.go b/g/encoding/gcompress/gcompress.go new file mode 100644 index 000000000..6875399db --- /dev/null +++ b/g/encoding/gcompress/gcompress.go @@ -0,0 +1,34 @@ +package gcompress + +import ( + "bytes" + "compress/zlib" + "io" +) + +// 进行zlib压缩 +func Zlib(data []byte) []byte { + if data == nil || len(data) < 13 { + return data + } + var in bytes.Buffer + w := zlib.NewWriter(&in) + w.Write(data) + w.Close() + return in.Bytes() +} + +// 进行zlib解压缩 +func UnZlib(data []byte) []byte { + if data == nil || len(data) < 13 { + return data + } + b := bytes.NewReader(data) + var out bytes.Buffer + r, err := zlib.NewReader(b) + if err != nil { + return nil + } + io.Copy(&out, r) + return out.Bytes() +} \ No newline at end of file diff --git a/g/encoding/gcrc32/gcrc32.go b/g/encoding/gcrc32/gcrc32.go new file mode 100644 index 000000000..470dbcc91 --- /dev/null +++ b/g/encoding/gcrc32/gcrc32.go @@ -0,0 +1,13 @@ +package gcrc32 + +import ( + "hash/crc32" +) + +func EncodeString(v string) uint32 { + return crc32.ChecksumIEEE([]byte(v)) +} + +func EncodeBytes(v []byte) uint32 { + return crc32.ChecksumIEEE(v) +} diff --git a/g/encoding/ghash/ghash.go b/g/encoding/ghash/ghash.go new file mode 100644 index 000000000..78d5186aa --- /dev/null +++ b/g/encoding/ghash/ghash.go @@ -0,0 +1,191 @@ +// 封装常用的hash函数 +package ghash + + +// BKDR Hash Function +func BKDRHash(str []byte) uint32 { + var seed uint32 = 131; // 31 131 1313 13131 131313 etc.. + var hash uint32 = 0; + for i := 0; i < len(str); i++ { + hash = hash * seed + uint32(str[i]) + } + return hash +} + +// BKDR Hash Function 64 +func BKDRHash64(str []byte) uint64 { + var seed uint64 = 131; // 31 131 1313 13131 131313 etc.. + var hash uint64 = 0; + for i := 0; i < len(str); i++ { + hash = hash * seed + uint64(str[i]) + } + return hash +} + +// SDBM Hash +func SDBMHash(str []byte) uint32 { + var hash uint32 = 0; + for i := 0; i < len(str); i++ { + // equivalent to: hash = 65599*hash + uint32(str[i]); + hash = uint32(str[i]) + (hash << 6) + (hash << 16) - hash; + } + return hash +} + +// SDBM Hash 64 +func SDBMHash64(str []byte) uint64 { + var hash uint64 = 0; + for i := 0; i < len(str); i++ { + // equivalent to: hash = 65599*hash + uint32(str[i]); + hash = uint64(str[i]) + (hash << 6) + (hash << 16) - hash; + } + return hash +} + +// RS Hash Function +func RSHash(str []byte) uint32 { + var b uint32 = 378551; + var a uint32 = 63689; + var hash uint32 = 0; + for i := 0; i < len(str); i++ { + hash = hash * a + uint32(str[i]); + a *= b; + } + return hash +} + +// RS Hash Function 64 +func RSHash64(str []byte) uint64 { + var b uint64 = 378551; + var a uint64 = 63689; + var hash uint64 = 0; + for i := 0; i < len(str); i++ { + hash = hash * a + uint64(str[i]); + a *= b; + } + return hash +} + +// JS Hash Function +func JSHash(str []byte) uint32 { + var hash uint32 = 1315423911; + for i := 0; i < len(str); i++ { + hash ^= ((hash << 5) + uint32(str[i]) + (hash >> 2)); + } + return hash +} + +// JS Hash Function 64 +func JSHash64(str []byte) uint64 { + var hash uint64 = 1315423911; + for i := 0; i < len(str); i++ { + hash ^= ((hash << 5) + uint64(str[i]) + (hash >> 2)); + } + return hash +} + +// P. J. Weinberger Hash Function +func PJWHash(str []byte) uint32 { + var BitsInUnignedInt uint32 = (4 * 8); + var ThreeQuarters uint32 = ((BitsInUnignedInt * 3) / 4); + var OneEighth uint32 = (BitsInUnignedInt / 8); + var HighBits uint32 = (0xFFFFFFFF) << (BitsInUnignedInt - OneEighth); + var hash uint32 = 0; + var test uint32 = 0; + for i := 0; i < len(str); i++ { + hash = (hash << OneEighth) + uint32(str[i]); + if test = hash & HighBits; test != 0 { + hash = ((hash ^ (test >> ThreeQuarters)) & (^HighBits + 1)); + } + } + return hash +} + +// P. J. Weinberger Hash Function 64 +func PJWHash64(str []byte) uint64 { + var BitsInUnignedInt uint64 = (4 * 8); + var ThreeQuarters uint64 = ((BitsInUnignedInt * 3) / 4); + var OneEighth uint64 = (BitsInUnignedInt / 8); + var HighBits uint64 = (0xFFFFFFFFFFFFFFFF) << (BitsInUnignedInt - OneEighth); + var hash uint64 = 0; + var test uint64 = 0; + for i := 0; i < len(str); i++ { + hash = (hash << OneEighth) + uint64(str[i]); + if test = hash & HighBits; test != 0 { + hash = ((hash ^ (test >> ThreeQuarters)) & (^HighBits + 1)); + } + } + return hash +} + +// ELF Hash Function +func ELFHash(str []byte) uint32 { + var hash uint32 = 0; + var x uint32 = 0; + for i := 0; i < len(str); i++ { + hash = (hash << 4) + uint32(str[i]); + if x = hash & 0xF0000000; x != 0 { + hash ^= (x >> 24); + hash &= ^x + 1; + } + } + return hash +} + +// ELF Hash Function 64 +func ELFHash64(str []byte) uint64 { + var hash uint64 = 0; + var x uint64 = 0; + for i := 0; i < len(str); i++ { + hash = (hash << 4) + uint64(str[i]); + if x = hash & 0xF000000000000000; x != 0 { + hash ^= (x >> 24); + hash &= ^x + 1; + } + } + return hash +} + +// DJB Hash Function +func DJBHash(str []byte) uint32 { + var hash uint32 = 5381; + for i := 0; i < len(str); i++ { + hash += (hash << 5) + uint32(str[i]); + } + return hash +} + +// DJB Hash Function 64 +func DJBHash64(str []byte) uint64 { + var hash uint64 = 5381; + for i := 0; i < len(str); i++ { + hash += (hash << 5) + uint64(str[i]); + } + return hash +} + +// AP Hash Function +func APHash(str []byte) uint32 { + var hash uint32 = 0; + for i := 0; i < len(str); i++ { + if ((i & 1) == 0) { + hash ^= ((hash << 7) ^ uint32(str[i]) ^ (hash >> 3)); + } else { + hash ^= (^((hash << 11) ^ uint32(str[i]) ^ (hash >> 5)) + 1); + } + } + return hash +} + +// AP Hash Function 64 +func APHash64(str []byte) uint64 { + var hash uint64 = 0; + for i := 0; i < len(str); i++ { + if ((i & 1) == 0) { + hash ^= ((hash << 7) ^ uint64(str[i]) ^ (hash >> 3)); + } else { + hash ^= (^((hash << 11) ^ uint64(str[i]) ^ (hash >> 5)) + 1); + } + } + return hash +} \ No newline at end of file diff --git a/g/encoding/ghtml/ghtml.go b/g/encoding/ghtml/ghtml.go new file mode 100644 index 000000000..5a4a84252 --- /dev/null +++ b/g/encoding/ghtml/ghtml.go @@ -0,0 +1,25 @@ +package ghtml + +import "strings" + +// 将html中的特殊标签转换为html转义标签 +func SpecialChars(s string) string { + return strings.NewReplacer( + "&", "&", + "<", "<", + ">", ">", + `"`, """, + "'", "'", + ).Replace(s) +} + +// 将html转义标签还原为html特殊标签 +func SpecialCharsDecode(s string) string { + return strings.NewReplacer( + "&", "&", + "<", "<", + ">", ">", + """, `"`, + "'", "'", + ).Replace(s) +} diff --git a/g/encoding/gjson/internal/json.go b/g/encoding/gjson/internal/json.go new file mode 100644 index 000000000..ea9089940 --- /dev/null +++ b/g/encoding/gjson/internal/json.go @@ -0,0 +1,436 @@ +package internal + +import ( + "fmt" + "errors" + "strings" + "strconv" +) + +// 这是一个使用go进行json语法解析的解析器，效率没有官方的json解析高，仅作学习参考 + +const ( + gJSON_CHAR_BRACE_LEFT = rune('{') + gJSON_CHAR_BRACE_RIGHT = rune('}') + gJSON_CHAR_BRACKET_LEFT = rune('[') + gJSON_CHAR_BRACKET_RIGHT = rune(']') + gJSON_CHAR_QUOTATION = rune('\\') + gJSON_CHAR_COMMA = rune(',') + gJSON_CHAR_COLON = rune(':') + gJSON_CHAR_DOUBLE_QUOTE_MARK = rune('"') +) + +const ( + gJSON_TOKEN_BRACE_LEFT = rune('{') + gJSON_TOKEN_BRACE_RIGHT = rune('}') + gJSON_TOKEN_BRACKET_LEFT = rune('[') + gJSON_TOKEN_BRACKET_RIGHT = rune(']') + gJSON_TOKEN_COMMA = rune(',') + gJSON_TOKEN_COLON = rune(':') + gJSON_TOKEN_STRING = rune('"') + gJSON_TOKEN_NUMBER = rune('0') +) + +// json关联数组(哈希表) +type JsonMap map[string]interface{} +// json索引数组(普通数组，从0开始索引) +type JsonArray []interface{} + +// JSON数据对象 +type gJsonNode struct { + m JsonMap + a JsonArray +} + +// JSON语义token +type gJsonToken struct { + token []rune // token字符串 + tokenType rune // token类型 + tokenindex int // token在原始字符串中的索引位置 +} + +// JSON解析结构对象 +type gJsonParser struct { + content []rune // 需要解析json字符串(通过string转换为[]rune) + tokens []gJsonToken // 存放解析content后的json token数组 + root *gJsonNode // json根节点 + pointer *gJsonNode // 指向当前正在解析的json节点 +} + +// 解析json字符串 +func Decode(j *string) (*gJsonParser, error) { + p := &gJsonParser{content:[]rune(*j)} + err := p.parse() + if err == nil { + return p, err + } else { + return nil, err + } +} + +// 判断所给字符串是否为数字 +func isNumeric(s string) bool { + for i :=0; i < len(s); i++ { + if s[i] < byte('0') || s[i] > byte('9') { + return false + } + } + return true +} + +// 获得一个键值对关联数组/哈希表，方便操作，不需要自己做类型转换 +// 注意，如果获取的值不存在，或者类型与json类型不匹配，那么将会返回nil +func (p *gJsonParser) GetMap(pattern string) JsonMap { + result := p.Get(pattern) + if result != nil { + if r, ok := result.(JsonMap); ok { + return r + } + } + return nil +} + +// 获得一个数组[]interface{}，方便操作，不需要自己做类型转换 +// 注意，如果获取的值不存在，或者类型与json类型不匹配，那么将会返回nil +func (p *gJsonParser) GetArray(pattern string) JsonArray { + result := p.Get(pattern) + if result != nil { + if r, ok := result.(JsonArray); ok { + return r + } + } + return nil +} + + +// 根据约定字符串方式访问json解析数据，参数形如： "items.name.first", "list.0" +// 返回的结果类型的interface{}，因此需要自己做类型转换 +// 如果找不到对应节点的数据，返回nil +func (p *gJsonParser) Get(pattern string) interface{} { + var result interface{} + pointer := p.root + array := strings.Split(pattern, ".") + length := len(array) + for i:= 0; i < length; i++ { + // 优先判断数组 + if isNumeric(array[i]) { + n, err := strconv.Atoi(array[i]) + if err == nil && len(pointer.a) > n { + if i == length - 1 { + result = pointer.a[n] + break; + } else { + if p, ok := pointer.a[n].(*gJsonNode); ok { + pointer = p + continue + } + } + } + } + // 其次判断哈希表，如果一个键在数组及map中均不存在，直接返回nil + if v, ok := pointer.m[array[i]]; ok { + if i == length - 1 { + result = v + } else { + if p, ok := v.(*gJsonNode); ok { + pointer = p + continue + } + } + } else { + return nil + } + } + // 处理结果，如果是gJsonNode类型，那么需要做转换 + if r, ok := result.(*gJsonNode); ok { + if len(r.m) < 1 { + return r.a + } else { + return r.m + } + } + return result +} + +// 遍历json字符串数组，并且判断转义 +func (p *gJsonParser) getNextChar(c rune, f int) int { + for i := f + 1; i < len(p.content); i++ { + if p.content[i] == c { + if i > 0 && p.content[i - 1] != gJSON_CHAR_QUOTATION { + return i + } + } else { + switch p.content[i] { + case gJSON_CHAR_DOUBLE_QUOTE_MARK: + r := p.getNextChar(gJSON_CHAR_DOUBLE_QUOTE_MARK, i) + if r > 0 { + i = r + } + } + } + } + return 0 +} + +// 判断字符是否为数字 +func (p *gJsonParser) isCharNumber(c rune) bool { + if c >= rune('0') && c <= rune('9') { + return true + } + return false +} + +// 按照json语法对保存的字符串进行解析 +func (p *gJsonParser) parse() error { + // 首先将字符串解析成token进行保存 + for i := 0; i < len(p.content); i++ { + if p.isCharNumber(p.content[i]) { + j := i + 1 + for ; j < len(p.content); j++ { + if !p.isCharNumber(p.content[j]) { + break; + } + } + p.tokens = append(p.tokens, gJsonToken { + token: p.content[i:j], + tokenType: gJSON_TOKEN_NUMBER, + tokenindex: i, + }) + i = j - 1 + } else { + switch p.content[i] { + case gJSON_CHAR_DOUBLE_QUOTE_MARK: + r := p.getNextChar(gJSON_CHAR_DOUBLE_QUOTE_MARK, i) + if r > 0 { + // 注意这里需要去掉字符串两边的双引号 + p.tokens = append(p.tokens, gJsonToken { + token: p.content[i+1:r], + tokenType: gJSON_TOKEN_STRING, + tokenindex: i, + }) + i = r + } + case gJSON_CHAR_COLON: + p.tokens = append(p.tokens, gJsonToken{token: p.content[i:i+1], tokenType: gJSON_TOKEN_COLON, tokenindex: i}) + case gJSON_CHAR_COMMA: + p.tokens = append(p.tokens, gJsonToken{token: p.content[i:i+1], tokenType: gJSON_TOKEN_COMMA, tokenindex: i}) + case gJSON_CHAR_BRACE_LEFT: + p.tokens = append(p.tokens, gJsonToken{token: p.content[i:i+1], tokenType: gJSON_TOKEN_BRACE_LEFT, tokenindex: i}) + case gJSON_CHAR_BRACE_RIGHT: + p.tokens = append(p.tokens, gJsonToken{token: p.content[i:i+1], tokenType: gJSON_TOKEN_BRACE_RIGHT, tokenindex: i}) + case gJSON_CHAR_BRACKET_LEFT: + p.tokens = append(p.tokens, gJsonToken{token: p.content[i:i+1], tokenType: gJSON_TOKEN_BRACKET_LEFT, tokenindex: i}) + case gJSON_CHAR_BRACKET_RIGHT: + p.tokens = append(p.tokens, gJsonToken{token: p.content[i:i+1], tokenType: gJSON_TOKEN_BRACKET_RIGHT, tokenindex: i}) + + default: + c := string(p.content[i]) + if c != " " && c != "\r" && c != "\n" && c != "\t" { + return errors.New(fmt.Sprintf("json parse error: invalid char '%s' at index %d", c, i)) + } + } + } + } + // 最后对解析后的token转换为go变量 + return p.parseTokenNodeToVar(0, len(p.tokens) - 1) +} + +// 获取json范围字符包含范围最右侧的索引位置 +func (p *gJsonParser)getTokenBorderRightIndex(token rune, from int) int { + switch token { + case gJSON_TOKEN_BRACE_LEFT: + leftCount := 0 + for i := from + 1; i < len(p.tokens); i++ { + if p.tokens[i].tokenType == gJSON_TOKEN_BRACE_LEFT { + leftCount ++ + } else if p.tokens[i].tokenType == gJSON_TOKEN_BRACE_RIGHT { + if leftCount < 1 { + return i + } else { + leftCount-- + } + } + } + case gJSON_CHAR_BRACKET_LEFT: + leftCount := 0 + for i := from + 1; i < len(p.tokens); i++ { + if p.tokens[i].tokenType == gJSON_CHAR_BRACKET_LEFT { + leftCount ++ + } else if p.tokens[i].tokenType == gJSON_CHAR_BRACKET_RIGHT { + if leftCount < 1 { + return i + } else { + leftCount-- + } + } + } + } + return 0 +} + +// 将解析过后的json token转换为go变量 +func (p *gJsonParser) parseTokenNodeToVar(left int, right int) error { + //fmt.Println("================================") + //for i := left; i <= right; i++ { + // fmt.Println(string(p.tokens[i].token)) + //} + for i := left; i <= right; i++ { + //fmt.Println(string(p.tokens[i].token)) + switch p.tokens[i].tokenType { + case gJSON_TOKEN_BRACE_LEFT: + fallthrough + case gJSON_TOKEN_BRACKET_LEFT: + node := newJsonNode() + // 判断根节点 + if p.root == nil { + p.root = node + p.pointer = node + } + // 判断层级关系 + borderRight := p.getTokenBorderRightIndex(p.tokens[i].tokenType, i) + if borderRight < 1 { + return errors.New(fmt.Sprintf("json parse error: unclosed tag '%s' at index %d", string(p.tokens[i].token), p.tokens[i].tokenindex)) + } + if i > 1 && ( + p.tokens[i-1].tokenType == gJSON_TOKEN_COLON && + p.tokens[i-2].tokenType == gJSON_TOKEN_STRING) { + // json赋值操作 + oldptr := p.pointer + k := string(p.tokens[i-2].token) + p.pointer.m[k] = node + p.pointer = node + err := p.parseTokenNodeToVar(i + 1, borderRight - 1) + if err != nil { + return err + } else { + i = borderRight + p.pointer = oldptr + } + + } else if i > 0 && ( + p.tokens[i-1].tokenType == gJSON_TOKEN_COMMA || + p.tokens[i-1].tokenType == gJSON_TOKEN_BRACE_LEFT || + p.tokens[i-1].tokenType == gJSON_TOKEN_BRACKET_LEFT) { + // json数组操作 + oldptr := p.pointer + p.pointer.a = append(p.pointer.a, node) + p.pointer = node + err := p.parseTokenNodeToVar(i + 1, borderRight - 1) + if err != nil { + return err + } else { + i = borderRight + p.pointer = oldptr + } + } else { + // json层级关系 + p.pointer = node + err := p.parseTokenNodeToVar(i + 1, borderRight - 1) + if err != nil { + return err + } else { + i = borderRight + } + } + + case gJSON_TOKEN_STRING: + fallthrough + case gJSON_TOKEN_NUMBER: + if i > 0 && p.tokens[i-1].tokenType == gJSON_TOKEN_COLON { + k := string(p.tokens[i-2].token) + v := string(p.tokens[i].token) + p.pointer.m[k] = v + } else if p.tokens[i+1].tokenType != gJSON_TOKEN_COLON { + p.pointer.a = append(p.pointer.a, string(p.tokens[i].token)) + } + + case gJSON_TOKEN_COLON: + if i < 1 || (p.tokens[i-1].tokenType != gJSON_TOKEN_STRING) { + return errors.New(fmt.Sprintf("json parse error: invalid charactar '%s' at index %d", string(p.tokens[i].token), p.tokens[i].tokenindex)) + } + + case gJSON_TOKEN_COMMA: + if (p.tokens[i+1].tokenType != gJSON_TOKEN_STRING && + p.tokens[i+1].tokenType != gJSON_TOKEN_NUMBER && + p.tokens[i+1].tokenType != gJSON_TOKEN_BRACE_LEFT && + p.tokens[i+1].tokenType != gJSON_TOKEN_BRACKET_LEFT) || + (i < 1 || ( + p.tokens[i-1].tokenType != gJSON_TOKEN_STRING && + p.tokens[i-1].tokenType != gJSON_TOKEN_NUMBER && + p.tokens[i-1].tokenType != gJSON_TOKEN_BRACE_RIGHT && + p.tokens[i-1].tokenType != gJSON_TOKEN_BRACKET_RIGHT)) { + return errors.New(fmt.Sprintf("json parse error: invalid charactar '%s' at index %d", string(p.tokens[i].token), p.tokens[i].tokenindex)) + } + } + } + return nil +} + +// 打印出所有的token(测试用) +func (p *gJsonParser)printTokens() { + for _, v := range p.tokens { + fmt.Println(string(v.token)) + } +} + +// 格式化打印根节点 +func (p *gJsonParser)Print() { + if len(p.root.m) > 0 { + fmt.Println("{") + } else { + fmt.Println("[") + } + p.printNode(p.pointer, "\t") + if len(p.root.m) > 0 { + fmt.Println("}") + } else { + fmt.Println("]") + } +} + +// 格式化打印根节点 +func (p *gJsonParser)printNode(n *gJsonNode, indent string) { + if len(n.m) > 0 { + for k, v := range n.m { + if t, ok := v.(*gJsonNode); ok { + if len(t.m) > 0 { + fmt.Printf("%v%v\t: {\n", indent, k) + p.printNode(t, indent + "\t") + fmt.Printf("%v}\n", indent) + } else { + fmt.Printf("%v%v\t: [\n", indent, k) + p.printNode(t, indent + "\t") + fmt.Printf("%v}\n", indent) + } + } else { + fmt.Printf("%v%v\t: %v\n", indent, k, v) + } + } + } + if len(n.a) > 0 { + for k, v := range n.a { + if t, ok := v.(*gJsonNode); ok { + if len(t.m) > 0 { + fmt.Printf("%v%v\t: {\n", indent, k) + p.printNode(t, indent + "\t") + fmt.Printf("%v}\n", indent) + } else { + fmt.Printf("%v%v\t: [\n", indent, k) + p.printNode(t, indent + "\t") + fmt.Printf("%v}\n", indent) + } + } else { + fmt.Printf("%v%v : %v\n", indent, k, v) + } + } + } +} + +// 创建一个json数据对象 +func newJsonNode() *gJsonNode { + return &gJsonNode { + m: make(map[string]interface{}), + a: make([]interface{}, 0), + } +} + diff --git a/g/encoding/gjson/json.go b/g/encoding/gjson/json.go new file mode 100644 index 000000000..9b609a299 --- /dev/null +++ b/g/encoding/gjson/json.go @@ -0,0 +1,216 @@ +package gjson + +import ( + "encoding/json" + "errors" + "strings" + "strconv" + "g/os/glog" + "fmt" +) + +// json解析结果存放数组 +type Json struct { + // 注意这是一个指针 + value *interface{} +} + +// 一个json变量 +type JsonVar interface{} + +// 编码go变量为json字符串，并返回json字符串指针 +func Encode (v interface{}) string { + s, err := json.Marshal(v) + if err != nil { + glog.Error("json marshaling failed: " + err.Error()) + return "" + } + r := string(s) + return r +} + +// 解码字符串为interface{}变量 +func Decode (s string) interface{} { + var v interface{} + if DecodeTo(s, &v) == nil { + return v + } + return nil +} + +// 解析json字符串为go变量，注意第二个参数为指针 +func DecodeTo (s string, v interface{}) error { + if err := json.Unmarshal([]byte(s), v); err != nil { + return errors.New("json unmarshaling failed: " + err.Error()) + } + return nil +} + +// 解析json字符串为gjson.Json对象，并返回操作对象指针 +func DecodeToJson (s string) *Json { + var result interface{} + if err := json.Unmarshal([]byte(s), &result); err != nil { + glog.Error("json unmarshaling failed: " + err.Error()) + return nil + } + return &Json{ &result } +} + +// 将变量转换为Json对象进行处理，该变量至少应当是一个map或者array，否者转换没有意义 +func NewJson(v *interface{}) *Json { + return &Json{ v } +} + +// 将指定的json内容转换为指定结构返回，查找失败或者转换失败，目标对象转换为nil +// 注意第二个参数需要给的是变量地址 +func (p *Json) GetToVar(pattern string, v interface{}) error { + r := p.Get(pattern) + if r != nil { + return DecodeTo(Encode(r), v) + } else { + v = nil + } + return nil +} + +// 获得一个键值对关联数组/哈希表，方便操作，不需要自己做类型转换 +// 注意，如果获取的值不存在，或者类型与json类型不匹配，那么将会返回nil +func (p *Json) GetMap(pattern string) map[string]interface{} { + result := p.Get(pattern) + if result != nil { + if r, ok := result.(map[string]interface{}); ok { + return r + } + } + return nil +} + +// 获得一个数组[]interface{}，方便操作，不需要自己做类型转换 +// 注意，如果获取的值不存在，或者类型与json类型不匹配，那么将会返回nil +func (p *Json) GetArray(pattern string) []interface{} { + result := p.Get(pattern) + if result != nil { + if r, ok := result.([]interface{}); ok { + return r + } + } + return nil +} + +// 返回指定json中的string +func (p *Json) GetString(pattern string) string { + result := p.Get(pattern) + if result != nil { + if r, ok := result.(string); ok { + return r + } + } + return "" +} + +// 返回指定json中的bool +func (p *Json) GetBool(pattern string) bool { + result := p.Get(pattern) + if result != nil { + str := fmt.Sprintf("%v", result) + if str != "" && str != "0" && str != "false" { + return true + } + } + return false +} + +// 返回指定json中的float64 +func (p *Json) GetFloat64(pattern string) float64 { + result := p.Get(pattern) + if result != nil { + if r, ok := result.(float64); ok { + return r + } + } + return 0 +} + +// 返回指定json中的float64->int +func (p *Json) GetInt(pattern string) int { + return int(p.GetFloat64(pattern)) +} + +// 返回指定json中的float64->int64 +func (p *Json) GetInt64(pattern string) int64 { + return int64(p.GetFloat64(pattern)) +} + +// 根据约定字符串方式访问json解析数据，参数形如： "items.name.first", "list.0" +// 返回的结果类型的interface{}，因此需要自己做类型转换 +// 如果找不到对应节点的数据，返回nil +func (p *Json) Get(pattern string) interface{} { + var result interface{} + pointer := p.value + array := strings.Split(pattern, ".") + length := len(array) + for i:= 0; i < length; i++ { + switch (*pointer).(type) { + case map[string]interface{}: + if v, ok := (*pointer).(map[string]interface{})[array[i]]; ok { + if i == length - 1 { + result = v + } else { + pointer = &v + } + } else { + return nil + } + case []interface{}: + if isNumeric(array[i]) { + n, err := strconv.Atoi(array[i]) + if err == nil && len((*pointer).([]interface{})) > n { + if i == length - 1 { + result = (*pointer).([]interface{})[n] + break; + } else { + pointer = &(*pointer).([]interface{})[n] + } + } + } else { + return nil + } + default: + return nil + } + } + return result +} + +// 转换为map[string]interface{}类型,如果转换失败，返回nil +func (p *Json) ToMap() map[string]interface{} { + pointer := p.value + switch (*pointer).(type) { + case map[string]interface{}: + return (*pointer).(map[string]interface{}) + default: + return nil + } +} + +// 转换为[]interface{}类型,如果转换失败，返回nil +func (p *Json) ToArray() []interface{} { + pointer := p.value + switch (*pointer).(type) { + case []interface{}: + return (*pointer).([]interface{}) + default: + return nil + } +} + + +// 判断所给字符串是否为数字 +func isNumeric(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] < byte('0') || s[i] > byte('9') { + return false + } + } + return true +} \ No newline at end of file diff --git a/g/encoding/gmd5/gmd5.go b/g/encoding/gmd5/gmd5.go new file mode 100644 index 000000000..e91a4765e --- /dev/null +++ b/g/encoding/gmd5/gmd5.go @@ -0,0 +1,48 @@ +package gmd5 + +import ( + "crypto/md5" + "fmt" + "encoding/json" + "reflect" + "os" + "io" + "g/os/glog" +) + +// 将任意类型的变量进行md5摘要(注意map等非排序变量造成的不同结果) +func Encode(v interface{}) string { + h := md5.New() + if "string" == reflect.TypeOf(v).String() { + h.Write([]byte(v.(string))) + } else { + b, err := json.Marshal(v) + if err != nil { + return "" + } else { + h.Write(b) + } + } + return fmt.Sprintf("%x", h.Sum(nil)) +} + +// 将字符串进行MD5哈希摘要计算 +func EncodeString(v string) string { + h := md5.New() + h.Write([]byte(v)) + return fmt.Sprintf("%x", h.Sum(nil)) +} + +// 将文件内容进行MD5哈希摘要计算 +func EncodeFile(path string) string { + f, e := os.Open(path) + if e != nil { + glog.Fatalln(e) + } + h := md5.New() + _, e = io.Copy(h, f) + if e != nil { + glog.Fatalln(e) + } + return fmt.Sprintf("%x", h.Sum(nil)) +} diff --git a/g/encoding/gsha1/gsha1.go b/g/encoding/gsha1/gsha1.go new file mode 100644 index 000000000..d00468b09 --- /dev/null +++ b/g/encoding/gsha1/gsha1.go @@ -0,0 +1,36 @@ +package gsha1 + +import ( + "crypto/sha1" + "encoding/hex" + "os" + "io" + "g/os/glog" + "g/encoding/gmd5" +) + +// 将任意类型的变量进行SHA摘要(注意map等非排序变量造成的不同结果) +// 内部使用了md5计算，因此效率会稍微差一些，更多情况请使用 EncodeString +func Encode(v interface{}) string { + return EncodeString(gmd5.Encode(v)) +} + +// 对字符串行SHA1摘要计算 +func EncodeString(s string) string { + r := sha1.Sum([]byte(s)) + return hex.EncodeToString(r[:]) +} + +// 对文件内容进行SHA1摘要计算 +func EncodeFile(path string) string { + f, e := os.Open(path) + if e != nil { + glog.Fatalln(e) + } + h := sha1.New() + _, e = io.Copy(h, f) + if e != nil { + glog.Fatalln(e) + } + return hex.EncodeToString(h.Sum(nil)) +} \ No newline at end of file diff --git a/g/g.go b/g/g.go new file mode 100644 index 000000000..467244122 --- /dev/null +++ b/g/g.go @@ -0,0 +1,6 @@ +package g + +// 框架信息 +const VERSION = "0.40" +const AUTHORS = "john" + diff --git a/g/net/ghttp/http.go b/g/net/ghttp/http.go new file mode 100644 index 000000000..a5180bd6c --- /dev/null +++ b/g/net/ghttp/http.go @@ -0,0 +1,107 @@ +package ghttp + +import ( + "net/http" + "time" + "crypto/tls" + "log" + "net/url" +) + +// http客户端 +type Client struct { + http.Client +} + +// http server结构体 +type Server struct { + server http.Server + config ServerConfig + handlerMap HandlerMap +} + +// 请求对象 +type ClientRequest struct { + http.Request + getvals *url.Values +} + +// 客户端请求结果对象 +type ClientResponse struct { + http.Response +} + +// 服务端请求返回对象 +type ServerResponse struct { + http.ResponseWriter +} + +// http回调函数 +type HandlerFunc func(*ClientRequest, *ServerResponse) + +// uri与回调函数的绑定记录表 +type HandlerMap map[string]HandlerFunc + +// HTTP Server 设置结构体 +type ServerConfig struct { + // HTTP Server基础字段 + Addr string // 监听IP和端口，监听本地所有IP使用":端口" + Handler http.Handler // 默认的处理函数 + TLSConfig *tls.Config // TLS配置 + ReadTimeout time.Duration + WriteTimeout time.Duration + IdleTimeout time.Duration + MaxHeaderBytes int // 最大的header长度 + ErrorLog *log.Logger // 错误日志的处理接口 + // gf 扩展信息字段 + IndexFiles []string // 默认访问的文件列表 + IndexFolder bool // 如果访问目录是否显示目录列表 + ServerAgent string // server agent + ServerRoot string // 服务器服务的本地目录根路径 +} + +// 默认HTTP Server +var defaultServerConfig = ServerConfig { + Addr : ":80", + Handler : nil, + ReadTimeout : 60 * time.Second, + WriteTimeout : 60 * time.Second, + IdleTimeout : 60 * time.Second, + MaxHeaderBytes : 1024, + IndexFiles : []string{"index.html", "index.htm"}, + IndexFolder : false, + ServerAgent : "gf", + ServerRoot : "", +} + +// 修改默认的http server配置 +func SetDefaultServerConfig (c ServerConfig) { + defaultServerConfig = c +} + +// 创建一个默认配置的HTTP Server(默认监听端口是80) +func NewServer() (*Server) { + return NewServerByConfig(defaultServerConfig) +} + +// 创建一个HTTP Server，返回指针 +func NewServerByAddr(addr string) (*Server) { + config := defaultServerConfig + config.Addr = addr + return NewServerByConfig(config) +} + +// 创建一个HTTP Server +func NewServerByAddrRoot(addr string, root string) (*Server) { + config := defaultServerConfig + config.Addr = addr + config.ServerRoot = root + return NewServerByConfig(config) +} + +// 根据输入配置创建一个http server对象 +func NewServerByConfig(s ServerConfig) (*Server) { + var server Server + server.SetConfig(s) + return &server +} \ No newline at end of file diff --git a/g/net/ghttp/http_client.go b/g/net/ghttp/http_client.go new file mode 100644 index 000000000..0f1877591 --- /dev/null +++ b/g/net/ghttp/http_client.go @@ -0,0 +1,122 @@ +package ghttp + +import ( + "net/http" + "strings" + "time" +) + +// http客户端对象指针 +func NewClient() (*Client) { + return &Client{} +} + +// 设置请求过期时间 +func (c *Client) SetTimeOut(t time.Duration) { + c.Timeout = t +} + +// GET请求 +func (c *Client) Get(url string) *ClientResponse { + return c.Request("GET", url, "") +} + +// PUT请求 +func (c *Client) Put(url, data string) *ClientResponse { + return c.Request("PUT", url, data) +} + +// POST请求提交数据 +func (c *Client) Post(url, data string) *ClientResponse { + resp, err := http.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data)) + if err != nil { + //glog.Println(err) + return nil + } + r := &ClientResponse{} + r.Response = *resp + return r +} + +// DELETE请求 +func (c *Client) Delete(url, data string) *ClientResponse { + return c.Request("DELETE", url, data) +} + +func (c *Client) Head(url, data string) *ClientResponse { + return c.Request("HEAD", url, data) +} + +func (c *Client) Patch(url, data string) *ClientResponse { + return c.Request("PATCH", url, data) +} + +func (c *Client) Connect(url, data string) *ClientResponse{ + return c.Request("CONNECT", url, data) +} + +func (c *Client) Options(url, data string) *ClientResponse{ + return c.Request("OPTIONS", url, data) +} + +func (c *Client) Trace(url, data string) *ClientResponse { + return c.Request("TRACE", url, data) +} + +// 请求并返回response对象 +func (c *Client) Request(method, url, data string) *ClientResponse { + req, err := http.NewRequest(strings.ToUpper(method), url, strings.NewReader(data)) + if err != nil { + //glog.Println("creating request failed: " + err.Error()) + return nil + } + resp, err := c.Do(req) + if err != nil { + //glog.Println("sending request failed: " + err.Error()) + return nil + } + r := &ClientResponse{} + r.Response = *resp + return r +} + + +func Get(url string) *ClientResponse { + return Request("GET", url, "") +} + +func Put(url, data string) *ClientResponse { + return Request("PUT", url, data) +} + +func Post(url, data string) *ClientResponse { + return Request("PUT", url, data) +} + +func Delete(url, data string) *ClientResponse { + return Request("DELETE", url, data) +} + +func Head(url, data string) *ClientResponse { + return Request("HEAD", url, data) +} + +func Patch(url, data string) *ClientResponse { + return Request("PATCH", url, data) +} + +func Connect(url, data string) *ClientResponse{ + return Request("CONNECT", url, data) +} + +func Options(url, data string) *ClientResponse{ + return Request("OPTIONS", url, data) +} + +func Trace(url, data string) *ClientResponse { + return Request("TRACE", url, data) +} + +func Request(method, url, data string) *ClientResponse { + return NewClient().Request(method, url, data) +} diff --git a/g/net/ghttp/http_controller.go b/g/net/ghttp/http_controller.go new file mode 100644 index 000000000..1b65b2181 --- /dev/null +++ b/g/net/ghttp/http_controller.go @@ -0,0 +1,29 @@ +package ghttp + +// 控制器基类 +type Controller struct { + Server *Server +} + +// 控制器接口 +type ControllerApi interface { + Get(r *ClientRequest, w *ServerResponse) + Put(r *ClientRequest, w *ServerResponse) + Post(r *ClientRequest, w *ServerResponse) + Delete(r *ClientRequest, w *ServerResponse) + Head(r *ClientRequest, w *ServerResponse) + Patch(r *ClientRequest, w *ServerResponse) + Connect(r *ClientRequest, w *ServerResponse) + Options(r *ClientRequest, w *ServerResponse) + Trace(r *ClientRequest, w *ServerResponse) +} + +func (c *Controller) Get(r *ClientRequest, w *ServerResponse) {} +func (c *Controller) Put(r *ClientRequest, w *ServerResponse) {} +func (c *Controller) Post(r *ClientRequest, w *ServerResponse) {} +func (c *Controller) Delete(r *ClientRequest, w *ServerResponse) {} +func (c *Controller) Head(r *ClientRequest, w *ServerResponse) {} +func (c *Controller) Patch(r *ClientRequest, w *ServerResponse) {} +func (c *Controller) Connect(r *ClientRequest, w *ServerResponse) {} +func (c *Controller) Options(r *ClientRequest, w *ServerResponse) {} +func (c *Controller) Trace(r *ClientRequest, w *ServerResponse) {} \ No newline at end of file diff --git a/g/net/ghttp/http_request.go b/g/net/ghttp/http_request.go new file mode 100644 index 000000000..551f735c3 --- /dev/null +++ b/g/net/ghttp/http_request.go @@ -0,0 +1,155 @@ +package ghttp + +import ( + "io/ioutil" + "g/encoding/gjson" +) + +// 获得get参数 +func (r *ClientRequest) GetQuery(k string) []string { + if r.getvals == nil { + values := r.URL.Query() + r.getvals = &values + } + if v, ok := (*r.getvals)[k]; ok { + return v + } + return nil +} + +func (r *ClientRequest) GetQueryString(k string) string { + v := r.GetQuery(k) + if v == nil { + return "" + } else { + return v[0] + } +} + +func (r *ClientRequest) GetQueryArray(k string) []string { + v := r.GetQuery(k) + if v == nil { + return nil + } else { + return v + } +} + +// 获取指定键名的关联数组，并且给定当指定键名不存在时的默认值 +func (r *ClientRequest) GetQueryMap(defaultMap map[string][]string) map[string][]string { + m := make(map[string][]string) + for k, v := range defaultMap { + v2 := r.GetQueryArray(k) + if v2 == nil { + m[k] = v + } else { + m[k] = v2 + } + } + return m +} + +// 获得post参数 +func (r *ClientRequest) GetPost(k string) []string { + if v, ok := r.PostForm[k]; ok { + return v + } + return nil +} + +func (r *ClientRequest) GetPostString(k string) string { + v := r.GetPost(k) + if v == nil { + return "" + } else { + return v[0] + } +} + +func (r *ClientRequest) GetPostArray(k string) []string { + v := r.GetPost(k) + if v == nil { + return nil + } else { + return v + } + return nil +} + +// 获取指定键名的关联数组，并且给定当指定键名不存在时的默认值 +func (r *ClientRequest) GetPostMap(defaultMap map[string][]string) map[string][]string { + m := make(map[string][]string) + for k, v := range defaultMap { + if v2, ok := r.PostForm[k]; ok { + m[k] = v2 + } else { + m[k] = v + } + } + return m +} + +// 获得post或者get提交的参数，如果有同名参数，那么按照get->post优先级进行覆盖 +func (r *ClientRequest) GetRequest(k string) []string { + v := r.GetQuery(k) + if v == nil { + return r.GetPost(k) + } + return v +} + +func (r *ClientRequest) GetRequestString(k string) string { + v := r.GetRequest(k) + if v == nil { + return "" + } else { + return v[0] + } +} + +func (r *ClientRequest) GetRequestArray(k string) []string { + v := r.GetRequest(k) + if v == nil { + return nil + } else { + return v + } + return nil +} + +// 获取指定键名的关联数组，并且给定当指定键名不存在时的默认值 +func (r *ClientRequest) GetRequestMap(defaultMap map[string][]string) map[string][]string { + m := make(map[string][]string) + for k, v := range defaultMap { + v2 := r.GetRequest(k) + if v2 != nil { + m[k] = v2 + } else { + m[k] = v + } + } + return m +} + + +// 获取原始请求输入字符串 +func (r *ClientRequest) GetRaw() string { + result, err := ioutil.ReadAll(r.Body) + if err != nil { + return "" + } else { + return string(result) + } +} + +// 获取原始请求输入字符串 +func (r *ClientRequest) GetJson() *gjson.Json { + data := r.GetRaw() + if data != "" { + return gjson.DecodeToJson(data) + } + return nil +} + + + diff --git a/g/net/ghttp/http_response.go b/g/net/ghttp/http_response.go new file mode 100644 index 000000000..4498552c9 --- /dev/null +++ b/g/net/ghttp/http_response.go @@ -0,0 +1,35 @@ +package ghttp + +import ( + "g/encoding/gjson" + "io/ioutil" + "g/os/glog" +) + +type ResponseJson struct { + Result int `json:"result"` + Message string `json:"message"` + Data interface{} `json:"data"` +} + +// 关闭返回的HTTP链接 +func (r *ClientResponse) Close() { + r.Response.Close = true + r.Body.Close() +} + +// 返回固定格式的json +func (r *ServerResponse) ResponseJson(result int, message string, data interface{}) { + r.Header().Set("Content-type", "application/json") + r.Write([]byte(gjson.Encode(ResponseJson{ result, message, data }))) +} + +// 获取返回的数据 +func (r *ClientResponse) ReadAll() string { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + glog.Println(err) + return "" + } + return string(body) +} \ No newline at end of file diff --git a/g/net/ghttp/http_server.go b/g/net/ghttp/http_server.go new file mode 100644 index 000000000..dfcc5b4f4 --- /dev/null +++ b/g/net/ghttp/http_server.go @@ -0,0 +1,165 @@ +package ghttp + +import ( + "net/http" + "strings" + "path/filepath" + "crypto/tls" + "time" + "log" + "regexp" + "g/os/glog" +) + +// 执行 +func (s *Server)Run() error { + // 底层http server配置 + if s.config.Handler == nil { + s.config.Handler = http.HandlerFunc(s.defaultHttpHandle) + } + s.server = http.Server { + Addr : s.config.Addr, + Handler : s.config.Handler, + ReadTimeout : s.config.ReadTimeout, + WriteTimeout : s.config.WriteTimeout, + IdleTimeout : s.config.IdleTimeout, + MaxHeaderBytes : s.config.MaxHeaderBytes, + } + // 执行端口监听 + err := s.server.ListenAndServe() + if err != nil { + glog.Fatalln(err) + } + return err +} + +// 获取默认的http server设置 +func (h Server)GetDefaultSetting() ServerConfig { + return defaultServerConfig +} + +// http server setting设置 +// 注意使用该方法进行http server配置时，需要配置所有的配置项，否则没有配置的属性将会默认变量为空 +func (s *Server)SetConfig(c ServerConfig) { + if c.Handler == nil { + c.Handler = http.HandlerFunc(s.defaultHttpHandle) + } + s.config = c + // 需要处理server root最后的目录分隔符号 + if s.config.ServerRoot != "" { + s.SetServerRoot(s.config.ServerRoot) + } + // 必需设置默认值的属性 + if len(s.config.IndexFiles) < 1 { + s.SetIndexFiles(defaultServerConfig.IndexFiles) + } + if s.config.ServerAgent == "" { + s.SetServerAgent(defaultServerConfig.ServerAgent) + } +} + +// 设置http server参数 - Addr +func (s *Server)SetAddr(addr string) { + s.config.Addr = addr +} + +// 设置http server参数 - Handler +func (s *Server)SetHandler(handler http.Handler) { + s.config.Handler = handler +} + +// 设置http server参数 - TLSConfig +func (s *Server)SetTLSConfig(tls *tls.Config) { + s.config.TLSConfig = tls +} + +// 设置http server参数 - ReadTimeout +func (s *Server)SetReadTimeout(t time.Duration) { + s.config.ReadTimeout = t +} + +// 设置http server参数 - WriteTimeout +func (s *Server)SetWriteTimeout(t time.Duration) { + s.config.WriteTimeout = t +} + +// 设置http server参数 - IdleTimeout +func (s *Server)SetIdleTimeout(t time.Duration) { + s.config.IdleTimeout = t +} + +// 设置http server参数 - MaxHeaderBytes +func (s *Server)SetMaxHeaderBytes(b int) { + s.config.MaxHeaderBytes = b +} + +// 设置http server参数 - ErrorLog +func (s *Server)SetErrorLog(logger *log.Logger) { + s.config.ErrorLog = logger +} + +// 设置http server参数 - IndexFiles +func (s *Server)SetIndexFiles(index []string) { + s.config.IndexFiles = index +} + +// 设置http server参数 - IndexFolder +func (s *Server)SetIndexFolder(index bool) { + s.config.IndexFolder = index +} + +// 设置http server参数 - ServerAgent +func (s *Server)SetServerAgent(agent string) { + s.config.ServerAgent = agent +} + +// 设置http server参数 - ServerRoot +func (s *Server)SetServerRoot(root string) { + s.config.ServerRoot = strings.TrimRight(root, string(filepath.Separator)) +} + +// 绑定URI到操作函数/方法 +// pattern的格式形如：/user/list, put:/user, delete:/user +// 支持RESTful的请求格式，具体业务逻辑由绑定的处理方法来执行 +func (s *Server)BindHandle(pattern string, handler HandlerFunc ) { + if s.handlerMap == nil { + s.handlerMap = make(HandlerMap) + } + key := "" + reg := regexp.MustCompile(`(\w+?)\s*:\s*(.+)`) + result := reg.FindStringSubmatch(pattern) + if len(result) > 1 { + key = strings.ToUpper(result[1]) + ":" + result[2] + } else { + key = strings.TrimSpace(pattern) + } + if _, ok := s.handlerMap[key]; ok { + panic("duplicated http server handler for: " + pattern) + } else { + s.handlerMap[key] = handler + } +} + +// 通过映射数组绑定URI到操作函数/方法 +func (s *Server)BindHandleByMap(m HandlerMap) { + for p, f := range m { + s.BindHandle(p, f) + } +} + +// 绑定控制器，控制器需要继承gmvc.ControllerBase对象并实现需要的REST方法 +func (s *Server)BindController(uri string, c ControllerApi) { + s.BindHandleByMap(HandlerMap{ + "GET:" + uri : c.Get, + "PUT:" + uri : c.Put, + "POST:" + uri : c.Post, + "DELETE:" + uri : c.Delete, + "PATCH:" + uri : c.Patch, + "HEAD:" + uri : c.Head, + "CONNECT:" + uri : c.Connect, + "OPTIONS:" + uri : c.Options, + "TRACE:" + uri : c.Trace, + }) +} + + diff --git a/g/net/ghttp/http_server_handle.go b/g/net/ghttp/http_server_handle.go new file mode 100644 index 000000000..80029caf8 --- /dev/null +++ b/g/net/ghttp/http_server_handle.go @@ -0,0 +1,113 @@ +package ghttp + +import ( + "net/http" + "strings" + "path/filepath" + "g/os/gfile" + "os" + "fmt" + "sort" + "net/url" + "g/encoding/ghtml" +) + +// 默认HTTP Server处理入口，底层默认使用了gorutine调用该接口 +func (s *Server)defaultHttpHandle(w http.ResponseWriter, r *http.Request) { + request := ClientRequest{} + response := ServerResponse {} + request.Request = *r + response.ResponseWriter = w + if f, ok := s.handlerMap[r.URL.Path]; ok { + f(&request, &response) + } else { + method := strings.ToUpper(r.Method) + if f, ok := s.handlerMap[method + ":" + r.URL.Path]; ok { + f(&request, &response) + } else { + s.serveFile(w, r) + } + } +} + +// 处理静态文件请求 +func (s *Server)serveFile(w http.ResponseWriter, r *http.Request) { + uri := r.URL.String() + if s.config.ServerRoot != "" { + // 获取文件的绝对路径 + path := strings.TrimRight(s.config.ServerRoot, string(filepath.Separator)) + path = path + uri + path = gfile.RealPath(path) + if (path != "") { + s.doServeFile(w, r, path) + } else { + s.NotFound(w, r) + } + } else { + s.NotFound(w, r) + } +} + +// http server静态文件处理 +func (s *Server)doServeFile(w http.ResponseWriter, r *http.Request, path string) { + f, err := os.Open(path) + if err != nil { + return + } + info, _ := f.Stat() + if info.IsDir() { + if len(s.config.IndexFiles) > 0 { + for _, file := range s.config.IndexFiles { + fpath := path + "/" + file + if gfile.Exists(fpath) { + f.Close() + s.doServeFile(w, r, fpath) + return + } + } + } + if s.config.IndexFolder { + s.listDir(w, f) + } else { + s.ResponseStatus(w, http.StatusForbidden) + } + } else { + http.ServeContent(w, r, info.Name(), info.ModTime(), f) + } + f.Close() +} + +// 目录列表 +func (s *Server)listDir(w http.ResponseWriter, f http.File) { + dirs, err := f.Readdir(-1) + if err != nil { + http.Error(w, "Error reading directory", http.StatusInternalServerError) + return + } + sort.Slice(dirs, func(i, j int) bool { return dirs[i].Name() < dirs[j].Name() }) + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + fmt.Fprintf(w, "
\n")
+    for _, d := range dirs {
+        name := d.Name()
+        if d.IsDir() {
+            name += "/"
+        }
+        url := url.URL{Path: name}
+        fmt.Fprintf(w, "%s\n", url.String(), ghtml.SpecialChars(name))
+    }
+    fmt.Fprintf(w, "
\n") +} + +// 返回http状态码，并使用默认配置的字符串返回信息 +func (s *Server)ResponseStatus(w http.ResponseWriter, code int) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(code) + fmt.Fprintln(w, http.StatusText(code)) +} + +// 404 +func (s *Server)NotFound(w http.ResponseWriter, r *http.Request) { + http.NotFound(w, r) +} \ No newline at end of file diff --git a/g/net/gip/ip.go b/g/net/gip/ip.go new file mode 100644 index 000000000..ac90a8b61 --- /dev/null +++ b/g/net/gip/ip.go @@ -0,0 +1,153 @@ +package gip + +import ( + "net" + "strconv" + "strings" + "regexp" + "fmt" +) + +// ip字符串转为整形 +func Ip2long(ipstr string) (ip uint32) { + r := `^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$` + reg, err := regexp.Compile(r) + if err != nil { + return + } + ips := reg.FindStringSubmatch(ipstr) + if ips == nil { + return + } + + ip1, _ := strconv.Atoi(ips[1]) + ip2, _ := strconv.Atoi(ips[2]) + ip3, _ := strconv.Atoi(ips[3]) + ip4, _ := strconv.Atoi(ips[4]) + + if ip1>255 || ip2>255 || ip3>255 || ip4 > 255 { + return + } + + ip += uint32(ip1 * 0x1000000) + ip += uint32(ip2 * 0x10000) + ip += uint32(ip3 * 0x100) + ip += uint32(ip4) + return +} + +// ip整形转为字符串 +func Long2ip(ip uint32) string { + return fmt.Sprintf("%d.%d.%d.%d", ip>>24, ip<<8>>24, ip<<16>>24, ip<<24>>24) +} + +// 获得ip的网段，例如：192.168.2.102 -> 192.168.2 +func GetSegment(ip string) string { + r := `^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$` + reg, err := regexp.Compile(r) + if err != nil { + return "" + } + ips := reg.FindStringSubmatch(ip) + if ips == nil { + return "" + } + return fmt.Sprintf("%s.%s.%s", ips[1], ips[2], ips[3]) +} + +// 解析地址，形如：192.168.1.1:80 -> 192.168.1.1, 80 +func ParseAddress(addr string) (string, int) { + r := `^(.+):(\d+)$` + reg, err := regexp.Compile(r) + if err != nil { + return "", 0 + } + result := reg.FindStringSubmatch(addr) + if result != nil { + i, _ := strconv.Atoi(result[2]) + return result[1], i + } + return "", 0 +} + +// 获取本地局域网ip列表 +func IntranetIP() (ips []string, err error) { + ips = make([]string, 0) + ifaces, e := net.Interfaces() + if e != nil { + return ips, e + } + for _, iface := range ifaces { + if iface.Flags&net.FlagUp == 0 { + continue // interface down + } + + if iface.Flags & net.FlagLoopback != 0 { + continue // loopback interface + } + + // ignore warden bridge + if strings.HasPrefix(iface.Name, "w-") { + continue + } + + addrs, e := iface.Addrs() + if e != nil { + return ips, e + } + + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + + if ip == nil || ip.IsLoopback() { + continue + } + + ip = ip.To4() + if ip == nil { + continue // not an ipv4 address + } + + ipStr := ip.String() + if IsIntranet(ipStr) { + ips = append(ips, ipStr) + } + } + } + return ips, nil +} + +// 判断所给ip是否为局域网ip +// A类 10.0.0.0--10.255.255.255 +// B类 172.16.0.0--172.31.255.255 +// C类 192.168.0.0--192.168.255.255 +func IsIntranet(ipStr string) bool { + // ip协议保留的局域网ip + if strings.HasPrefix(ipStr, "10.") || strings.HasPrefix(ipStr, "192.168.") { + return true + } + if strings.HasPrefix(ipStr, "172.") { + // 172.16.0.0 - 172.31.255.255 + arr := strings.Split(ipStr, ".") + if len(arr) != 4 { + return false + } + + second, err := strconv.ParseInt(arr[1], 10, 64) + if err != nil { + return false + } + + if second >= 16 && second <= 31 { + return true + } + } + + return false +} diff --git a/g/net/gscanner/scanner.go b/g/net/gscanner/scanner.go new file mode 100644 index 000000000..fb7de9017 --- /dev/null +++ b/g/net/gscanner/scanner.go @@ -0,0 +1,87 @@ +// 局域网端口扫描 +package gscanner + +import ( + "net" + "g/net/gip" + "fmt" + "errors" + "sync" + "time" +) + +type scanner struct { + timeout time.Duration +} + +// 初始化一个扫描器 +func New() *scanner { + return &scanner{ + 6*time.Second, + } +} + +// 设置超时时间，注意这个时间是每一次扫描的超时时间，而不是总共的超时时间 +func (s *scanner) SetTimeout(t time.Duration) *scanner { + s.timeout = t + return s +} + +// 异步TCP扫描网段及端口，如果扫描的端口是打开的，那么将链接给定给回调函数进行调用 +// 注意startIp和endIp需要是同一个网段，否则会报错,并且回调函数不会执行 +func (s *scanner) ScanIp(startIp string, endIp string, port int, callback func(net.Conn)) error { + if callback == nil { + return errors.New("callback function should not be nil") + } + var waitGroup sync.WaitGroup + startIplong := gip.Ip2long(startIp) + endIplong := gip.Ip2long(endIp) + result := endIplong - startIplong + if startIplong == 0 || endIplong == 0 { + return errors.New("invalid startip or endip: ipv4 string should be given") + } + if result < 0 || result > 255 { + return errors.New("invalid startip and endip: startip and endip should be in the same ip segment") + } + + for i := startIplong; i <= endIplong; i++ { + waitGroup.Add(1) + go func(ip string) { + //fmt.Println("scanning:", ip) + // 这里必需设置超时时间 + conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", ip, port), s.timeout) + if err == nil { + callback(conn) + conn.Close() + } + //fmt.Println("scanning:", ip, "done") + waitGroup.Done() + }(gip.Long2ip(i)) + } + waitGroup.Wait() + return nil +} + +// 扫描目标主机打开的端口列表 +func (s *scanner) ScanPort(ip string, callback func(net.Conn)) error { + if callback == nil { + return errors.New("callback function should not be nil") + } + + var waitGroup sync.WaitGroup + for i := 0; i <= 65536; i++ { + waitGroup.Add(1) + //fmt.Println("scanning:", i) + go func(port int) { + conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", ip, port), s.timeout) + if err == nil { + callback(conn) + conn.Close() + } + waitGroup.Done() + }(i) + } + waitGroup.Wait() + return nil +} + diff --git a/g/net/gsmtp/smtp.go b/g/net/gsmtp/smtp.go new file mode 100644 index 000000000..18b79e724 --- /dev/null +++ b/g/net/gsmtp/smtp.go @@ -0,0 +1,78 @@ +package gsmtp + +import ( + "encoding/base64" + "fmt" + "net/smtp" + "strings" +) + +// 示例： +// s := smtp.New("smtp.exmail.qq.com:25", "notify@a.com", "password") +// glog.Println(s.SendMail("notify@a.com", "ulric@b.com;rain@c.com", "这是subject", "这是body,red")) + +type Smtp struct { + Address string + Username string + Password string +} + +func New(address, username, password string) *Smtp { + return &Smtp{ + Address: address, + Username: username, + Password: password, + } +} + +func (this *Smtp) SendMail(from, tos, subject, body string, contentType ...string) error { + if this.Address == "" { + return fmt.Errorf("address is necessary") + } + + hp := strings.Split(this.Address, ":") + if len(hp) != 2 { + return fmt.Errorf("address format error") + } + + arr := strings.Split(tos, ";") + count := len(arr) + safeArr := make([]string, 0, count) + for i := 0; i < count; i++ { + if arr[i] == "" { + continue + } + safeArr = append(safeArr, arr[i]) + } + + if len(safeArr) == 0 { + return fmt.Errorf("tos invalid") + } + + tos = strings.Join(safeArr, ";") + + b64 := base64.NewEncoding("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/") + + header := make(map[string]string) + header["From"] = from + header["To"] = tos + header["Subject"] = fmt.Sprintf("=?UTF-8?B?%s?=", b64.EncodeToString([]byte(subject))) + header["MIME-Version"] = "1.0" + + ct := "text/plain; charset=UTF-8" + if len(contentType) > 0 && contentType[0] == "html" { + ct = "text/html; charset=UTF-8" + } + + header["Content-Type"] = ct + header["Content-Transfer-Encoding"] = "base64" + + message := "" + for k, v := range header { + message += fmt.Sprintf("%s: %s\r\n", k, v) + } + message += "\r\n" + b64.EncodeToString([]byte(body)) + + auth := smtp.PlainAuth("", this.Username, this.Password, hp[0]) + return smtp.SendMail(this.Address, auth, from, strings.Split(tos, ";"), []byte(message)) +} \ No newline at end of file diff --git a/g/net/gtcp/tcp.go b/g/net/gtcp/tcp.go new file mode 100644 index 000000000..5ed787a69 --- /dev/null +++ b/g/net/gtcp/tcp.go @@ -0,0 +1,29 @@ +package gtcp + +import ( + "net" + "g/os/glog" +) + +// tcp server结构体 +type gTcpServer struct { + address string + listener *net.TCPListener + handler func (net.Conn) +} + +// 创建一个tcp server对象 +func NewServer (address string, handler func (net.Conn)) *gTcpServer { + tcpaddr, err := net.ResolveTCPAddr("tcp4", address) + if err != nil { + glog.Fatalln(err) + return nil + } + listen, err := net.ListenTCP("tcp", tcpaddr) + if err != nil { + glog.Fatalln(err) + return nil + } + return &gTcpServer{ address, listen, handler} +} + diff --git a/g/net/gtcp/tcp_server.go b/g/net/gtcp/tcp_server.go new file mode 100644 index 000000000..f803d914b --- /dev/null +++ b/g/net/gtcp/tcp_server.go @@ -0,0 +1,21 @@ +package gtcp + +import ( + "g/os/glog" +) + +// 执行监听 +func (s *gTcpServer) Run() { + if s == nil || s.listener == nil { + glog.Println("start running failed: socket address bind failed") + return + } + for { + conn, err := s.listener.Accept() + if err != nil { + glog.Error(err) + } else if conn != nil { + go s.handler(conn) + } + } +} diff --git a/g/net/gudp/udp.go b/g/net/gudp/udp.go new file mode 100644 index 000000000..e2a7000f2 --- /dev/null +++ b/g/net/gudp/udp.go @@ -0,0 +1,29 @@ +package gudp + +import ( + "net" + "log" +) + +// tcp server结构体 +type gUdpServer struct { + address string + listener *net.UDPConn + handler func (*net.UDPConn) +} + +// 创建一个tcp server对象 +func NewServer (address string, handler func (*net.UDPConn)) *gUdpServer { + tcpaddr, err := net.ResolveUDPAddr("udp4", address) + if err != nil { + glog.Println(err) + return nil + } + listen, err := net.ListenUDP("udp", tcpaddr) + if err != nil { + glog.Println(err) + return nil + } + return &gUdpServer{ address, listen, handler} +} + diff --git a/g/net/gudp/udp_server.go b/g/net/gudp/udp_server.go new file mode 100644 index 000000000..58aeb9c9a --- /dev/null +++ b/g/net/gudp/udp_server.go @@ -0,0 +1,19 @@ +package gudp + +import "log" + +// 执行监听 +func (s *gUdpServer) Run() { + if s == nil || s.listener == nil { + glog.Println("start running failed: socket address bind failed") + return + } + if s.handler == nil { + glog.Println("start running failed: socket handler not defined") + return + } + for { + s.handler(s.listener) + } + +} diff --git a/g/net/gurl/url.go b/g/net/gurl/url.go new file mode 100644 index 000000000..c71231966 --- /dev/null +++ b/g/net/gurl/url.go @@ -0,0 +1,13 @@ +package gurl + +import "net/url" + +// url encode string, is + not %20 +func Encode(str string) string { + return url.QueryEscape(str) +} + +// url decode string +func Decode(str string) (string, error) { + return url.QueryUnescape(str) +} diff --git a/g/os/gcache/gcache.go b/g/os/gcache/gcache.go new file mode 100644 index 000000000..f13eae377 --- /dev/null +++ b/g/os/gcache/gcache.go @@ -0,0 +1,233 @@ +package gcache + +import ( + "sync" + "g/util/gtime" + "time" + "g/encoding/ghash" +) + +const ( + gCACHE_GROUP_SIZE = 4 // 缓存分区大小，不能超过uint8的最大值 +) + +type Cache struct { + sync.RWMutex + m map[uint8]*CacheMap // 以分区大小数字作为索引 +} + +type CacheMap struct { + sync.RWMutex + deleted bool // 对象是否已删除，以便判断停止goroutine + m map[string]CacheItem // 键值对 +} + +type CacheItem struct { + v interface{} // 缓存键值 + e int64 // 过期时间 +} + +// 全局缓存管理对象 +var cache *Cache = New() + +// Cache对象按照缓存键名首字母做了分组 +func New() *Cache { + c := &Cache { + m : make(map[uint8]*CacheMap), + } + var i uint8 = 0 + for ; i < gCACHE_GROUP_SIZE; i++ { + m := &CacheMap { + m : make(map[string]CacheItem), + } + c.m[i] = m + go m.autoClearLoop() + } + return c +} + +// (使用全局KV缓存对象)设置kv缓存键值对，过期时间单位为毫秒 +func Set(k string, v interface{}, expired int64) { + cache.Set(k, v, expired) +} + +// (使用全局KV缓存对象)批量设置kv缓存键值对，过期时间单位为毫秒 +func BatchSet(m map[string]interface{}, expired int64) { + cache.BatchSet(m, expired) +} + +// (使用全局KV缓存对象)获取指定键名的值 +func Get(k string) interface{} { + return cache.Get(k) +} + +// (使用全局KV缓存对象)删除指定键值对 +func Remove(k string) { + cache.Remove(k) +} + +// (使用全局KV缓存对象)批量删除指定键值对 +func BatchRemove(l []string) { + cache.BatchRemove(l) +} + +// 设置kv缓存键值对，过期时间单位为毫秒 +func (c *Cache) Set(k string, v interface{}, expired int64) { + c.RLock() + c.m[c.getIndex(k)].Set(k, v, expired) + c.RUnlock() +} + +// 批量设置 +func (c *Cache) BatchSet(m map[string]interface{}, expired int64) { + c.RLock() + for k, v := range m { + c.m[c.getIndex(k)].Set(k, v, expired) + } + c.RUnlock() +} + +// 获取指定键名的值 +func (c *Cache) Get(k string) interface{} { + c.RLock() + r := c.m[c.getIndex(k)].Get(k) + c.RUnlock() + return r +} + +// 删除指定键值对 +func (c *Cache) Remove(k string) { + c.RLock() + c.m[c.getIndex(k)].Remove(k) + c.RUnlock() +} + +// 批量删除键值对 +func (c *Cache) BatchRemove(l []string) { + c.RLock() + for _, k := range l { + c.m[c.getIndex(k)].Remove(k) + } + c.RUnlock() +} + +// 获得所有的键名，组成字符串数组返回 +func (c *Cache) Keys() []string { + l := make([]string, 0) + c.RLock() + for _, cm := range c.m { + cm.RLock() + for k2, _ := range cm.m { + l = append(l, k2) + } + cm.RUnlock() + } + c.RUnlock() + return l +} + +// 获得所有的值，组成数组返回 +func (c *Cache) Values() []interface{} { + l := make([]interface{}, 0) + c.RLock() + for _, cm := range c.m { + cm.RLock() + for _, v2 := range cm.m { + l = append(l, v2.v) + } + cm.RUnlock() + } + c.RUnlock() + return l +} + +// 获得缓存对象的键值对数量 +func (c *Cache) Size() int { + var size int + c.RLock() + for _, cm := range c.m { + cm.RLock() + size += len(cm.m) + cm.RUnlock() + } + c.RUnlock() + return size +} + +// 删除缓存对象 +func (c *Cache) Close() { + c.RLock() + for _, cm := range c.m { + cm.Lock() + cm.deleted = true + cm.Unlock() + } + c.RUnlock() + + c.Lock() + c.m = nil + c.Unlock() +} + +// 计算缓存的索引 +func (c *Cache) getIndex(k string) uint8 { + return uint8(ghash.BKDRHash([]byte(k)) % gCACHE_GROUP_SIZE) +} + +// 设置kv缓存键值对，过期时间单位为毫秒 +func (cm *CacheMap) Set(k string, v interface{}, expired int64) { + var e int64 + if expired > 0 { + e = gtime.Millisecond() + int64(expired) + } + cm.Lock() + cm.m[k] = CacheItem{v: v, e: e} + cm.Unlock() +} + +// 获取指定键名的值 +func (cm *CacheMap) Get(k string) interface{} { + var v interface{} + cm.RLock() + if r, ok := cm.m[k]; ok { + if r.e > 0 && r.e < gtime.Millisecond() { + v = nil + } else { + v = r.v + } + } + cm.RUnlock() + return v +} + +// 删除指定键值对 +func (cm *CacheMap) Remove(k string) { + cm.Lock() + delete(cm.m, k) + cm.Unlock() +} + +// 自动清理过期键值对(每间隔60秒执行) +func (cm *CacheMap) autoClearLoop() { + for !cm.deleted { + expired := make([]string, 0) + cm.RLock() + for k, v := range cm.m { + if v.e > 0 && v.e < gtime.Millisecond() { + expired = append(expired, k) + } + } + cm.RUnlock() + + if len(expired) > 0 { + cm.Lock() + for _, k := range expired { + delete(cm.m, k) + } + cm.Unlock() + } + time.Sleep(60 * time.Second) + } +} + + diff --git a/g/os/gconsole/gconsole.go b/g/os/gconsole/gconsole.go new file mode 100644 index 000000000..6a2e2bcf8 --- /dev/null +++ b/g/os/gconsole/gconsole.go @@ -0,0 +1,145 @@ +package gconsole + +import ( + "os" + "regexp" + "errors" + "strconv" + "strings" +) + +// 命令行参数列表 +type gConsoleValue struct { + values []string +} + +// 命令行选项列表 +type gConsoleOption struct { + options map[string]string +} + +// 终端管理对象(全局) +var Value gConsoleValue // console终端参数-命令参数列表 +var Option gConsoleOption // console终端参数-选项参数列表 +var cmdFuncMap = make(map[string]func()) // 终端命令及函数地址对应表 + +// 检查并初始化console参数，在包加载的时候触发 +// 初始化时执行，不影响运行时性能 +func init() { + Option.options = make(map[string]string) + reg := regexp.MustCompile(`\-\-{0,1}(\w+?)=(.+)`) + for i := 0; i < len(os.Args); i++ { + result := reg.FindStringSubmatch(os.Args[i]) + if len(result) > 1 { + Option.options[result[1]] = result[2] + } else { + Value.values = append(Value.values, os.Args[i]) + } + } +} + +// 返回所有的命令行参数values +func (c gConsoleValue) GetAll() []string { + return c.values +} + +// 返回所有的命令行参数options +func (c gConsoleOption) GetAll() map[string]string { + return c.options +} + +// 获得一条指定索引位置的value参数 +func (c gConsoleValue) Get(index uint8) string { + if index < uint8(len(c.values)) { + return c.values[index] + } + return "" +} + +// 类型转换 +func (c gConsoleValue) GetInt(key uint8) int { + v := c.Get(key) + if v != "" { + i, _ := strconv.Atoi(v) + return i + } + return 0 +} + +// 类型转换bool +func (c gConsoleValue) GetBool(key uint8) bool { + v := c.Get(key) + v = strings.ToLower(v) + if v != "" && v != "0" && v != "false" { + return true + } + return false +} + +// 获得一条指定索引位置的option参数 +func (c gConsoleOption) Get(key string) string { + option, ok := c.options[key] + if ok { + return option + } + return "" +} + +// 类型转换int +func (c gConsoleOption) GetInt(key string) int { + v := c.Get(key) + if v != "" { + i, _ := strconv.Atoi(v) + return i + } + return 0 +} + +// 类型转换bool +func (c gConsoleOption) GetBool(key string) bool { + v := c.Get(key) + v = strings.ToLower(v) + if v != "" && v != "0" && v != "false" { + return true + } + return false +} + +// 绑定命令行参数及对应的命令函数，注意参数是函数的内存地址 +// 如果操作失败返回错误信息 +func BindHandle (cmd string, f func()) error { + _, ok := cmdFuncMap[cmd] + if ok { + return errors.New("duplicated handle for command:" + cmd) + } else { + cmdFuncMap[cmd] = f + return nil + } +} + +// 执行命令对应的函数 +func RunHandle (cmd string) error { + handle, ok := cmdFuncMap[cmd] + if ok { + handle() + return nil + } else { + return errors.New("no handle found for command:" + cmd) + } +} + +// 自动识别命令参数并执行命令参数对应的函数 +func AutoRun () error { + cmd := Value.Get(1); + if cmd != "" { + if handle, ok := cmdFuncMap[cmd]; ok { + handle() + return nil + } else { + return errors.New("no handle found for command:" + cmd) + } + } else { + return errors.New("no command found") + } + +} diff --git a/g/os/gfile/gfile.go b/g/os/gfile/gfile.go new file mode 100644 index 000000000..452ef2f16 --- /dev/null +++ b/g/os/gfile/gfile.go @@ -0,0 +1,426 @@ +package gfile + +import ( + "os" + "path/filepath" + "io" + "io/ioutil" + "sort" + "fmt" + "time" + "strings" + "bytes" + "os/exec" + "errors" + "os/user" + "runtime" +) + +// 封装了常用的文件操作方法，如需更详细的文件控制，请查看官方os包 + +// 文件分隔符 +var Separator = string(filepath.Separator) + +// 给定文件的绝对路径创建文件 +func Mkdir(path string) error { + err := os.MkdirAll(path, os.ModePerm) + if err != nil { + return err + } + return nil +} + +// 给定文件的绝对路径创建文件 +func Create(path string) error { + f, err := os.Create(path) + if err != nil { + return err + } + f.Close() + return nil +} + +// 打开文件 +func Open(path string) (*os.File, error) { + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0755) + if err != nil { + return nil, err + } + return f, nil +} + +// 打开文件 +func OpenWithFlag(path string, flag int) (*os.File, error) { + f, err := os.OpenFile(path, flag, 0755) + if err != nil { + return nil, err + } + return f, nil +} + +// 判断所给路径文件/文件夹是否存在 +func Exists(path string) bool { + _, err := os.Stat(path) + if err != nil { + if os.IsExist(err) { + return true + } + return false + } + return true +} + +// 判断所给路径是否为文件夹 +func IsDir(path string) bool { + s, err := os.Stat(path) + if err != nil { + return false + } + return s.IsDir() +} + +// 判断所给路径是否为文件 +func IsFile(path string) bool { + return !IsDir(path) +} + +// 获取文件或目录信息 +func Info(path string) *os.FileInfo { + info, err := os.Stat(path) + if err != nil { + return nil + } + return &info +} + +// 修改时间 +func MTime(path string) int64 { + f, e := os.Stat(path) + if e != nil { + return 0 + } + return f.ModTime().Unix() +} + +// 文件大小(bytes) +func Size(path string) int64 { + f, e := os.Stat(path) + if e != nil { + return 0 + } + return f.Size() +} + +// 格式化文件大小 +func ReadableSize(path string) string { + return FormatSize(float64(Size(path))) +} + +// 格式化文件大小 +func FormatSize(raw float64) string { + var t float64 = 1024 + var d float64 = 1 + + if raw < t { + return fmt.Sprintf("%.2fB", raw/d) + } + + d *= 1024 + t *= 1024 + + if raw < t { + return fmt.Sprintf("%.2fK", raw/d) + } + + d *= 1024 + t *= 1024 + + if raw < t { + return fmt.Sprintf("%.2fM", raw/d) + } + + d *= 1024 + t *= 1024 + + if raw < t { + return fmt.Sprintf("%.2fG", raw/d) + } + + d *= 1024 + t *= 1024 + + if raw < t { + return fmt.Sprintf("%.2fT", raw/d) + } + + d *= 1024 + t *= 1024 + + if raw < t { + return fmt.Sprintf("%.2fP", raw/d) + } + + return "TooLarge" +} + +// 文件移动/重命名 +func Move(src string, dst string) error { + return os.Rename(src, dst) +} + + +// 文件移动/重命名 +func Rename(src string, dst string) error { + return Move(src, dst) +} + +// 文件复制 +func Copy(src string, dst string) error { + srcFile, err := os.Open(src) + if err != nil { + return err + } + dstFile, err := os.Create(dst) + if err != nil { + return err + } + _, err = io.Copy(dstFile, srcFile) + if err != nil { + return err + } + err = dstFile.Sync() + if err != nil { + return err + } + srcFile.Close() + dstFile.Close() + return nil +} + +// 文件/目录删除 +func Remove(path string) error { + return os.RemoveAll(path) +} + +// 文件是否可 +func IsReadable(path string) bool { + result := true + file, err := os.OpenFile(path, os.O_RDONLY, 0666) + if err != nil { + result = false + } + file.Close() + return result +} + +// 文件是否可写 +func IsWritable(path string) bool { + result := true + if IsDir(path) { + // 如果是目录，那么创建一个临时文件进行写入测试 + tfile := strings.TrimRight(path, Separator) + Separator + string(time.Now().UnixNano()) + err := Create(tfile) + if err != nil || !Exists(tfile){ + result = false + } else { + Remove(tfile) + } + } else { + // 如果是文件，那么判断文件是否可打开 + file, err := os.OpenFile(path, os.O_WRONLY, 0666) + if err != nil { + result = false + } + file.Close() + } + return result +} + +// 修改文件/目录权限 +func Chmod(path string, mode os.FileMode) error { + return os.Chmod(path, mode) +} + +// 打开目录，并返回其下一级子目录名称列表，按照文件名称大小写进行排序 +func ScanDir(path string) []string { + f, err := os.Open(path) + if err != nil { + return nil + } + + list, err := f.Readdirnames(-1) + f.Close() + if err != nil { + return nil + } + sort.Slice(list, func(i, j int) bool { return list[i] < list[j] }) + return list +} + +// 将所给定的路径转换为绝对路径 +// 并判断文件路径是否存在，如果文件不存在，那么返回空字符串 +func RealPath(path string) string { + p, err := filepath.Abs(path) + if err != nil { + return "" + } + if !Exists(p) { + return "" + } + return p +} + +// (文本)读取文件内容 +func GetContents(path string) string { + return string(GetBinContents(path)) +} + +// (二进制)读取文件内容 +func GetBinContents(path string) []byte { + data, err := ioutil.ReadFile(path) + if err != nil { + return nil + } + return data +} + +// 写入文件内容 +func putContents(path string, data []byte, flag int, perm os.FileMode) error { + // 支持目录递归创建 + dir := Dir(path) + if !Exists(dir) { + Mkdir(dir) + } + // 创建/打开文件 + f, err := os.OpenFile(path, flag, perm) + if err != nil { + return err + } + defer f.Close() + n, err := f.Write(data) + if err != nil { + return err + } else if n < len(data) { + return io.ErrShortWrite + } + return nil +} + +// (文本)写入文件内容 +func PutContents(path string, content string) error { + return putContents(path, []byte(content), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) +} + +// (文本)追加内容到文件末尾 +func PutContentsAppend(path string, content string) error { + return putContents(path, []byte(content), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755) +} + +// (二进制)写入文件内容 +func PutBinContents(path string, content []byte) error { + return putContents(path, content, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) +} + +// (二进制)追加内容到文件末尾 +func PutBinContentsAppend(path string, content []byte) error { + return putContents(path, content, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755) +} + + +// 获取当前执行文件的绝对路径 +func SelfPath() string { + p, _ := filepath.Abs(os.Args[0]) + return p +} + +// 获取当前执行文件的目录绝对路径 +func SelfDir() string { + return filepath.Dir(SelfPath()) +} + +// 获取指定文件路径的文件名称 +func Basename(path string) string { + return filepath.Base(path) +} + +// 获取指定文件路径的目录地址绝对路径 +func Dir(path string) string { + return filepath.Dir(path) +} + +// 获取指定文件路径的文件扩展名 +func Ext(path string) string { + return filepath.Ext(path) +} + +// 获取用户主目录 +func Home() (string, error) { + u, err := user.Current() + if nil == err { + return u.HomeDir, nil + } + if "windows" == runtime.GOOS { + return homeWindows() + } + return homeUnix() +} + +func homeUnix() (string, error) { + if home := os.Getenv("HOME"); home != "" { + return home, nil + } + var stdout bytes.Buffer + cmd := exec.Command("sh", "-c", "eval echo ~$USER") + cmd.Stdout = &stdout + if err := cmd.Run(); err != nil { + return "", err + } + + result := strings.TrimSpace(stdout.String()) + if result == "" { + return "", errors.New("blank output when reading home directory") + } + + return result, nil +} + +func homeWindows() (string, error) { + drive := os.Getenv("HOMEDRIVE") + path := os.Getenv("HOMEPATH") + home := drive + path + if drive == "" || path == "" { + home = os.Getenv("USERPROFILE") + } + if home == "" { + return "", errors.New("HOMEDRIVE, HOMEPATH, and USERPROFILE are blank") + } + + return home, nil +} + +// 获得文件内容下一个指定字节的位置 +func GetNextCharOffset(file *os.File, char string, start int64) int64 { + c := []byte(char)[0] + b := make([]byte, 1) + o := start + for { + _, err := file.ReadAt(b, o) + if err != nil { + return 0 + } + if b[0] == c { + return o + } + o++ + } + return 0 +} + +// 获得文件内容中两个offset之间的内容 [start, end) +func GetBinContentByTwoOffsets(file *os.File, start int64, end int64) []byte { + buffer := make([]byte, end - start) + if _, err := file.ReadAt(buffer, start); err != nil { + return nil + } + return buffer +} \ No newline at end of file diff --git a/g/os/gfilepool/gfilepool.go b/g/os/gfilepool/gfilepool.go new file mode 100644 index 000000000..c71c605c7 --- /dev/null +++ b/g/os/gfilepool/gfilepool.go @@ -0,0 +1,115 @@ +package gfilepool + +import ( + "os" + "g/core/types/glist" + "g/util/gtime" + "time" + "g/core/types/gmap" + "strconv" +) + +// 文件指针池 +type Pool struct { + path string // 文件绝对路径 + flag int // 文件打开标识 + list *glist.SafeList // 可用/闲置的文件指针链表 + idlemax int // 闲置最大时间，超过该时间则被系统回收(秒) + closed bool // 连接池是否已关闭 +} + +// 文件指针池指针 +type File struct { + pool *Pool // 所属池 + file *os.File // 指针对象 + expire int64 // 过期时间 +} + +// 全局指针池，expire < 0表示不过期，expire = 0表示使用完立即回收，expire > 0表示超时回收 +var pools *gmap.StringInterfaceMap = gmap.NewStringInterfaceMap() + +// 获得文件对象，并自动创建指针池 +func OpenWithPool(path string, flag int, expire int) (*File, error) { + key := path + strconv.Itoa(flag) + strconv.Itoa(expire) + result := pools.Get(key) + if result != nil { + return result.(*Pool).File() + } + pool := New(path, flag, expire) + pools.Set(key, pool) + return pool.File() +} + +// 创建一个文件指针池，expire < 0表示不过期，expire = 0表示使用完立即回收，expire > 0表示超时回收 +func New(path string, flag int, expire int) *Pool { + r := &Pool { + path : path, + flag : flag, + list : glist.NewSafeList(), + idlemax : expire, + } + // 独立的线程执行过期清理工作 + if expire != -1 { + go func(p *Pool) { + for !p.closed { + r := p.list.Front() + if r != nil && r.Value != nil { + f := r.Value.(*File) + if f.expire <= gtime.Second() { + if f.file != nil { + f.file.Close() + } + p.list.Remove(r) + continue + } + } + time.Sleep(3 * time.Second) + } + }(r) + } + return r +} + +// 获得一个文件打开指针 +func (p *Pool) File() (*File, error) { + if p.list.Len() > 0 { + for { + r := p.list.PopBack() + if r != nil { + f := r.(*File) + if f.expire > gtime.Second() { + return f, nil + } else if f.file != nil { + f.file.Close() + f.file = nil + } + } else { + break; + } + } + } + file, err := os.OpenFile(p.path, p.flag, 0755) + if err != nil { + return nil, err + } + return &File { + pool : p, + file : file, + }, nil +} + +// 关闭指针池 +func (p *Pool) Close() { + p.closed = true +} + +// 获得底层文件指针 +func (f *File) File() *os.File { + return f.file +} + +// 关闭指针链接(软关闭) +func (f *File) Close() { + f.expire = gtime.Second() + int64(f.pool.idlemax) + f.pool.list.PushFront(f) +} \ No newline at end of file diff --git a/g/os/gfilespace/gfilespace.go b/g/os/gfilespace/gfilespace.go new file mode 100644 index 000000000..ea8e5e23f --- /dev/null +++ b/g/os/gfilespace/gfilespace.go @@ -0,0 +1,186 @@ +// 文件空间管理， 可用于文件碎片空间维护及再利用，支持自动合并连续碎片空间 + +package gfilespace + +import ( + "sync" + "g/core/types/gbtree" +) + +// 文件空间管理结构体 +type Space struct { + mu sync.RWMutex // 并发操作锁 + blocks *gbtree.BTree // 所有的空间块构建的B+树 + sizetr *gbtree.BTree // 空间块大小构建的B+树 + sizemap map[int]*gbtree.BTree // 按照空间块大小构建的索引哈希表，便于检索，每个表项是一个B+树 +} + +// 文件空闲块 +type Block struct { + index int // 文件偏移量 + size int // 区块大小(byte) +} + +// 用于B+树的接口具体实现定义 +func (block *Block) Less(item gbtree.Item) bool { + if block.index < item.(*Block).index { + return true + } + return false +} + +// 创建一个空间管理器 +func New() *Space { + return &Space { + blocks : gbtree.New(10), + sizetr : gbtree.New(5), + sizemap : make(map[int]*gbtree.BTree), + } +} + +// 添加空闲空间到管理器 +func (space *Space) addBlock(index int, size int) { + block := &Block{index, size} + + // 插入进全局树 + space.blocks.ReplaceOrInsert(block) + + // 插入进入索引表 + space.insertIntoSizeMap(block) + + // 对插入的数据进行合并检测 + space.checkMerge(block) +} + +// 获取指定block的前一项block +func (space *Space) getPrevBlock(block *Block) *Block { + var pblock *Block = nil + space.blocks.DescendLessOrEqual(block, func(item gbtree.Item) bool { + if item.(*Block).index != block.index { + pblock = item.(*Block) + return false + } + return true + }) + return pblock +} + +// 获取指定block的后一项block +func (space *Space) getNextBlock(block *Block) *Block { + var nblock *Block = nil + space.blocks.AscendGreaterOrEqual(block, func(item gbtree.Item) bool { + if item.(*Block).index != block.index { + nblock = item.(*Block) + return false + } + return true + }) + return nblock +} + +// 获取指定block的前一项block size +func (space *Space) getPrevBlockSize(size int) int { + psize := 0 + space.sizetr.DescendLessOrEqual(gbtree.Int(size), func(item gbtree.Item) bool { + if int(item.(gbtree.Int)) != size { + psize = int(item.(gbtree.Int)) + return false + } + return true + }) + return psize +} + +// 获取指定block的后一项block size +func (space *Space) getNextBlockSize(size int) int { + nsize := 0 + space.sizetr.AscendGreaterOrEqual(gbtree.Int(size), func(item gbtree.Item) bool { + if int(item.(gbtree.Int)) != size { + nsize = int(item.(gbtree.Int)) + return false + } + return true + }) + return nsize +} + +// 内部按照索引检查合并 +func (space *Space) checkMerge(block *Block) { + // 首先检查插入空间块的前一项往后是否可以合并，如果当前合并失败后，才会判断当前插入项和后续的空间块合并 + if b := space.checkMergeOfTwoBlock(space.getPrevBlock(block), block); b.index == block.index { + // 其次检查插入空间块的当前项往后是否可以合并 + space.checkMergeOfTwoBlock(block, space.getNextBlock(block)) + } +} + +// 连续检测两个空间块的合并，返回最后一个无法合并的空间块指针 +func (space *Space) checkMergeOfTwoBlock(pblock, block *Block) *Block { + if pblock == nil { + return block + } + if block == nil { + return pblock + } + for { + if pblock.index + int(pblock.size) >= block.index { + space.removeBlock(block) + // 判断是否需要更新大小 + if pblock.index + int(pblock.size) < block.index + int(block.size) { + space.removeFromSizeMap(pblock) + pblock.size = block.index + block.size - pblock.index + space.insertIntoSizeMap(pblock) + } + block = space.getNextBlock(pblock) + if block == nil { + return pblock + } + } else { + break + } + } + return block +} + +// 插入空间块到索引表 +func (space *Space) insertIntoSizeMap(block *Block) { + tree, ok := space.sizemap[block.size] + if !ok { + tree = gbtree.New(10) + space.sizemap[block.size] = tree + } + tree.ReplaceOrInsert(block) + + // 插入空间块大小记录表 + space.sizetr.ReplaceOrInsert(gbtree.Int(block.size)) +} + + +// 删除一项 +func (space *Space) removeBlock(block *Block) { + space.blocks.Delete(block) + space.removeFromSizeMap(block) +} + +// 从索引表中删除对应的空间块 +func (space *Space) removeFromSizeMap(block *Block) { + if tree, ok := space.sizemap[block.size]; ok { + tree.Delete(block) + // 数据数据为空，那么删除该项哈希记录 + if tree.Len() == 0 { + delete(space.sizemap, block.size) + space.sizetr.Delete(gbtree.Int(block.size)) + } + } +} + +// 获得碎片偏移量 +func (block *Block) Index() int { + return block.index +} + +// 获得碎片大小 +func (block *Block) Size() int { + return block.size +} + + diff --git a/g/os/gfilespace/gfilespace_api.go b/g/os/gfilespace/gfilespace_api.go new file mode 100644 index 000000000..ddf77cb35 --- /dev/null +++ b/g/os/gfilespace/gfilespace_api.go @@ -0,0 +1,177 @@ +package gfilespace + +import ( + "g/core/types/gbtree" + "g/encoding/gbinary" +) + +// 添加空闲空间到管理器 +func (space *Space) AddBlock(index int, size int) { + if size <= 0 { + return + } + space.mu.Lock() + defer space.mu.Unlock() + + space.addBlock(index, size) +} + +// 申请空间，返回文件地址及大小，返回成功后则在管理器中删除该空闲块 +func (space *Space) GetBlock(size int) (int, int) { + if size <= 0 { + return -1, 0 + } + space.mu.Lock() + defer space.mu.Unlock() + + for { + if tree, ok := space.sizemap[size]; ok { + if r := tree.Min(); r != nil { + block := r.(*Block) + space.removeBlock(block) + return block.index, block.size + } + } + size = space.getNextBlockSize(size) + if size == 0 { + break + } + } + return -1, 0 +} + +// 删除指定索引位置的空间块 +func (space *Space) RemoveBlock(index int) { + space.mu.Lock() + defer space.mu.Unlock() + + space.removeBlock(&Block{index, 0}) +} + +// 给定的空间块*整块*是否包含在管理器中 +func (space *Space) Contains(index int, size int) bool { + block := &Block{index, size} + if r := space.blocks.Get(block); r != nil { + if r.(*Block).size >= size { + return true + } + } else { + pblock := space.getPrevBlock(block) + if pblock != nil && (pblock.index <= index && (pblock.index + pblock.size) >= (index + size)) { + return true + } + } + return false +} + +// 获取索引最小的空间块 +func (space *Space) GetMinBlock() *Block { + space.mu.RLock() + defer space.mu.RUnlock() + var block *Block + space.blocks.Ascend(func(item gbtree.Item) bool { + block = item.(*Block) + return true + }) + return block +} + +// 获取索引最大的空间块 +func (space *Space) GetMaxBlock() *Block { + space.mu.RLock() + defer space.mu.RUnlock() + var block *Block + space.blocks.Descend(func(item gbtree.Item) bool { + block = item.(*Block) + return true + }) + return block +} + +// 获得所有的碎片空间，按照index升序排序 +func (space *Space) GetAllBlocks() []Block { + space.mu.RLock() + defer space.mu.RUnlock() + blocks := make([]Block, 0) + space.blocks.Ascend(func(item gbtree.Item) bool { + blocks = append(blocks, *(item.(*Block))) + return true + }) + return blocks +} + +// 获得所有的碎片空间大小列表，按照size升序排序 +func (space *Space) GetAllSizes() []uint { + space.mu.RLock() + defer space.mu.RUnlock() + sizes := make([]uint, 0) + space.sizetr.Ascend(func(item gbtree.Item) bool { + sizes = append(sizes, uint(item.(gbtree.Int))) + return true + }) + return sizes +} + +// 获取当前空间管理器中最大的空闲块大小 +func (space *Space) GetMaxSize() int { + space.mu.RLock() + defer space.mu.RUnlock() + + if item := space.sizetr.Max(); item != nil { + return int(item.(gbtree.Int)) + } + return 0 +} + +// 计算总的空闲空间大小 +func (space *Space) SumSize() int { + space.mu.RLock() + defer space.mu.RUnlock() + size := 0 + space.blocks.Ascend(func(item gbtree.Item) bool { + size += item.(*Block).size + return true + }) + return size +} + +// 获取空间块的数量 +func (space *Space) Len() int { + space.mu.RLock() + defer space.mu.RUnlock() + + return space.blocks.Len() +} + +// 导出空间块数据 +func (space *Space) Export() []byte { + space.mu.RLock() + defer space.mu.RUnlock() + + content := make([]byte, 0) + space.blocks.Ascend(func(item gbtree.Item) bool { + block := item.(*Block) + content = append(content, gbinary.EncodeInt64(int64(block.Index()))...) + content = append(content, gbinary.EncodeInt32(int32(block.Size()))...) + return true + }) + + return content +} + +// 导入空间块数据 +func (space *Space) Import(content []byte) { + space.mu.Lock() + defer space.mu.Unlock() + + for i := 0; i < len(content); i += 12 { + space.addBlock( + int(gbinary.DecodeToInt64(content[i : i + 8])), + int(gbinary.DecodeToInt32(content[i + 8 : i + 12])), + ) + } +} + + + + diff --git a/g/os/glog/glog.go b/g/os/glog/glog.go new file mode 100644 index 000000000..9ddc27ca1 --- /dev/null +++ b/g/os/glog/glog.go @@ -0,0 +1,413 @@ +package glog + +import ( + "sync" + "os" + "io" + "strings" + "reflect" + "path/filepath" + "time" + "fmt" + "g/os/gfile" +) + +type Logger struct { + mutex sync.RWMutex + logio io.Writer + debug bool // 是否允许输出DEBUG信息 + logpath string // 日志写入的目录路径 + lastlogdate string // 上一次写入日志的日期，例如: 2006-01-02 +} + +// 默认的日志对象 +var logger = New() + +// 新建自定义的日志操作对象 +func New() *Logger { + return &Logger{ + debug : true, + } +} + +func SetLogPath(path string) { + logger.SetLogPath(path) +} + +func SetDebug(debug bool) { + logger.SetDebug(debug) +} + +func GetLogPath() string { + return logger.GetLogPath() +} + +func Print(v ...interface{}) { + logger.Print(v ...) +} + +func Printf(format string, v ...interface{}) { + logger.Printf(format, v ...) +} + +func Println(v ...interface{}) { + logger.Println(v ...) +} + +func Printfln(format string, v ...interface{}) { + logger.Printfln(format, v ...) +} + +func Fatal(v ...interface{}) { + logger.Fatal(v ...) +} + +func Fatalf(format string, v ...interface{}) { + logger.Fatalf(format, v ...) +} + +func Fatalln(v ...interface{}) { + logger.Fatalln(v ...) +} + +func Fatalfln(format string, v ...interface{}) { + logger.Fatalfln(format, v ...) +} + +func Panic(v ...interface{}) { + logger.Panic(v ...) +} + +func Panicf(format string, v ...interface{}) { + logger.Panicf(format, v ...) +} + +func Panicln(v ...interface{}) { + logger.Panicln(v ...) +} + +func Panicfln(format string, v ...interface{}) { + logger.Panicfln(format, v ...) +} + +func Info(v ...interface{}) { + logger.Info(v...) +} + +func Debug(v ...interface{}) { + logger.Debug(v...) +} + +func Notice(v ...interface{}) { + logger.Notice(v...) +} + +func Warning(v ...interface{}) { + logger.Warning(v...) +} + +func Error(v ...interface{}) { + logger.Error(v...) +} + +func Critical(v ...interface{}) { + logger.Critical(v...) +} + +func Infof(format string, v ...interface{}) { + logger.Infof(format, v...) +} + +func Debugf(format string, v ...interface{}) { + logger.Debugf(format, v...) +} + +func Noticef(format string, v ...interface{}) { + logger.Noticef(format, v...) +} + +func Warningf(format string, v ...interface{}) { + logger.Warningf(format, v...) +} + +func Errorf(format string, v ...interface{}) { + logger.Errorf(format, v...) +} + +func Criticalf(format string, v ...interface{}) { + logger.Criticalf(format, v...) +} + +func Infofln(format string, v ...interface{}) { + logger.Infofln(format, v...) +} + +func Debugfln(format string, v ...interface{}) { + logger.Debugfln(format, v...) +} + +func Noticefln(format string, v ...interface{}) { + logger.Noticefln(format, v...) +} + +func Warningfln(format string, v ...interface{}) { + logger.Warningfln(format, v...) +} + +func Errorfln(format string, v ...interface{}) { + logger.Errorfln(format, v...) +} + +func Criticalfln(format string, v ...interface{}) { + logger.Criticalfln(format, v...) +} + +func (l *Logger) GetLogIO() io.Writer { + l.mutex.RLock() + r := l.logio + l.mutex.RUnlock() + return r +} + +func (l *Logger) GetDebug() bool { + l.mutex.RLock() + r := l.debug + l.mutex.RUnlock() + return r +} + +func (l *Logger) GetLogPath() string { + l.mutex.RLock() + r := l.logpath + l.mutex.RUnlock() + return r +} + +func (l *Logger) GetLastLogDate() string { + l.mutex.RLock() + r := l.lastlogdate + l.mutex.RUnlock() + return r +} + +func (l *Logger) SetLogIO(w io.Writer) { + l.mutex.RLock() + l.logio = w + l.mutex.RUnlock() +} + +func (l *Logger) SetDebug(debug bool) { + l.mutex.Lock() + l.debug = debug + l.mutex.Unlock() +} + +// 设置日志文件的存储目录路径 +func (l *Logger) SetLogPath(path string) { + l.mutex.Lock() + l.logpath = strings.TrimRight(path, string(filepath.Separator)) + l.mutex.Unlock() + // 重新检查日志io对象 + l.checkLogIO() +} + +// 检查文件名称是否已经过期 +func (l *Logger) checkLogIO() { + date := time.Now().Format("2006-01-02") + if date != l.GetLastLogDate() { + path := l.GetLogPath() + if path != "" { + if !gfile.Exists(path) { + err := gfile.Mkdir(path) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return + } + } + if !gfile.IsWritable(path) { + fmt.Fprintln(os.Stderr, path + " is no writable for current user") + return + } + + l.mutex.Lock() + fname := date + ".log" + fpath := l.logpath + string(filepath.Separator) + fname + fio, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0755) + if err == nil && fio != nil { + if l.logio != nil && reflect.TypeOf(l.logio).String() == "*os.File" { + l.logio.(*os.File).Close() + } + l.logio = fio + } else { + fmt.Fprintln(os.Stderr, err) + } + l.mutex.Unlock() + } + } +} + +// 核心打印数据方法(标准输出) +func (l *Logger) stdPrint(s string) { + l.checkLogIO() + l.mutex.Lock() + if l.logio == nil { + fmt.Fprint(os.Stdout, l.format(s)) + } else { + fmt.Fprint(l.logio, l.format(s)) + } + l.mutex.Unlock() +} + +// 核心打印数据方法(标准错误) +func (l *Logger) errPrint(s string) { + l.checkLogIO() + l.mutex.Lock() + if l.logio == nil { + fmt.Fprint(os.Stderr, l.format(s)) + } else { + fmt.Fprint(l.logio, l.format(s)) + } + l.mutex.Unlock() +} + +func (l *Logger) format(s string) string { + return time.Now().Format("2006-01-02 15:04:05 ") + s +} + +func (l *Logger) Print(v ...interface{}) { + l.stdPrint(fmt.Sprint(v...)) +} + +func (l *Logger) Printf(format string, v ...interface{}) { + l.stdPrint(fmt.Sprintf(format, v...)) +} + +func (l *Logger) Println(v ...interface{}) { + l.stdPrint(fmt.Sprintln(v...)) +} + +func (l *Logger) Printfln(format string, v ...interface{}) { + l.stdPrint(fmt.Sprintf(format + "\n", v...)) +} + +func (l *Logger) Fatal(v ...interface{}) { + l.errPrint(fmt.Sprint(v...)) + os.Exit(1) +} + +func (l *Logger) Fatalf(format string, v ...interface{}) { + l.errPrint(fmt.Sprintf(format, v...)) + os.Exit(1) +} + +func (l *Logger) Fatalln(v ...interface{}) { + l.errPrint(fmt.Sprintln(v...)) + os.Exit(1) +} + +func (l *Logger) Fatalfln(format string, v ...interface{}) { + l.errPrint(fmt.Sprintf(format + "\n", v...)) + os.Exit(1) +} + +func (l *Logger) Panic(v ...interface{}) { + s := fmt.Sprint(v...) + l.errPrint(s) + panic(s) +} + +func (l *Logger) Panicf(format string, v ...interface{}) { + s := fmt.Sprintf(format, v...) + l.errPrint(s) + panic(s) +} + +func (l *Logger) Panicln(v ...interface{}) { + s := fmt.Sprintln(v...) + l.errPrint(s) + panic(s) +} + +func (l *Logger) Panicfln(format string, v ...interface{}) { + s := fmt.Sprintf(format + "\n", v...) + l.errPrint(s) + panic(s) +} + +func (l *Logger) Info(v ...interface{}) { + l.stdPrint("[INFO] " + fmt.Sprintln(v...)) +} + +func (l *Logger) Debug(v ...interface{}) { + if l.GetDebug() { + l.stdPrint("[DEBU] " + fmt.Sprintln(v...)) + } +} + +func (l *Logger) Notice(v ...interface{}) { + l.errPrint("[NOTI] " + fmt.Sprintln(v...)) +} + +func (l *Logger) Warning(v ...interface{}) { + l.errPrint("[WARN] " + fmt.Sprintln(v...)) +} + +func (l *Logger) Error(v ...interface{}) { + l.errPrint("[ERRO] " + fmt.Sprintln(v...)) +} + +func (l *Logger) Critical(v ...interface{}) { + l.errPrint("[CRIT] " + fmt.Sprintln(v...)) +} + +func (l *Logger) Infof(format string, v ...interface{}) { + l.stdPrint("[INFO] " + fmt.Sprintf(format, v...)) +} + +func (l *Logger) Debugf(format string, v ...interface{}) { + if l.GetDebug() { + l.stdPrint("[DEBU] " + fmt.Sprintf(format, v...)) + } +} + +func (l *Logger) Noticef(format string, v ...interface{}) { + l.errPrint("[NOTI] " + fmt.Sprintf(format, v...)) +} + +func (l *Logger) Warningf(format string, v ...interface{}) { + l.errPrint("[WARN] " + fmt.Sprintf(format, v...)) +} + +func (l *Logger) Errorf(format string, v ...interface{}) { + l.errPrint("[ERRO] " + fmt.Sprintf(format, v...)) +} + +func (l *Logger) Criticalf(format string, v ...interface{}) { + l.errPrint("[CRIT] " + fmt.Sprintf(format, v...)) +} + +func (l *Logger) Infofln(format string, v ...interface{}) { + l.stdPrint("[INFO] " + fmt.Sprintf(format, v...) + "\n") +} + +func (l *Logger) Debugfln(format string, v ...interface{}) { + if l.GetDebug() { + l.stdPrint("[DEBU] " + fmt.Sprintf(format, v...) + "\n") + } +} + +func (l *Logger) Noticefln(format string, v ...interface{}) { + l.errPrint("[NOTI] " + fmt.Sprintf(format, v...) + "\n") +} + +func (l *Logger) Warningfln(format string, v ...interface{}) { + l.errPrint("[WARN] " + fmt.Sprintf(format, v...) + "\n") +} + +func (l *Logger) Errorfln(format string, v ...interface{}) { + l.errPrint("[ERRO] " + fmt.Sprintf(format, v...) + "\n") +} + +func (l *Logger) Criticalfln(format string, v ...interface{}) { + l.errPrint("[CRIT] " + fmt.Sprintf(format, v...) + "\n") +} \ No newline at end of file diff --git a/g/os/gmmap/gmmap_solaris.go b/g/os/gmmap/gmmap_solaris.go new file mode 100644 index 000000000..74c7089a3 --- /dev/null +++ b/g/os/gmmap/gmmap_solaris.go @@ -0,0 +1,38 @@ +// from https://github.com/influxdata/influxdb/tree/master/pkg/mmap +package gmmap + +import ( + "os" + "syscall" + "golang.org/x/sys/unix" +) + +func Map(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + return nil, err + } else if fi.Size() == 0 { + return nil, nil + } + + data, err := unix.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) + if err != nil { + return nil, err + } + + return data, nil +} + +// Unmap closes the memory-map. +func Unmap(data []byte) error { + if data == nil { + return nil + } + return unix.Munmap(data) +} \ No newline at end of file diff --git a/g/os/gmmap/gmmap_test.go b/g/os/gmmap/gmmap_test.go new file mode 100644 index 000000000..221a00dfb --- /dev/null +++ b/g/os/gmmap/gmmap_test.go @@ -0,0 +1,21 @@ +package gmmap + +import ( + "bytes" + "io/ioutil" + "testing" + "g/os/gmmap" +) + +func TestMap(t *testing.T) { + data, err := gmmap.Map("mmap_test.go") + if err != nil { + t.Fatalf("Open: %v", err) + } + + if exp, err := ioutil.ReadFile("mmap_test.go"); err != nil { + t.Fatalf("ioutil.ReadFile: %v", err) + } else if !bytes.Equal(data, exp) { + t.Fatalf("got %q\nwant %q", string(data), string(exp)) + } +} \ No newline at end of file diff --git a/g/os/gmmap/gmmap_unix.go b/g/os/gmmap/gmmap_unix.go new file mode 100644 index 000000000..c14a311e3 --- /dev/null +++ b/g/os/gmmap/gmmap_unix.go @@ -0,0 +1,37 @@ +// from https://github.com/influxdata/influxdb/tree/master/pkg/mmap +package gmmap + +import ( + "os" + "syscall" +) + +// Map memory-maps a file. +func Map(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + return nil, err + } else if fi.Size() == 0 { + return nil, nil + } + + data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) + if err != nil { + return nil, err + } + return data, nil +} + +// Unmap closes the memory-map. +func Unmap(data []byte) error { + if data == nil { + return nil + } + return syscall.Munmap(data) +} diff --git a/g/os/gmmap/gmmap_windows.go b/g/os/gmmap/gmmap_windows.go new file mode 100644 index 000000000..abad52004 --- /dev/null +++ b/g/os/gmmap/gmmap_windows.go @@ -0,0 +1,47 @@ +// from https://github.com/influxdata/influxdb/tree/master/pkg/mmap +package gmmap + +import ( + "os" + "syscall" + "unsafe" +) + +// Map memory-maps a file. +func Map(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + return nil, err + } else if fi.Size() == 0 { + return nil, nil + } + + lo, hi := uint32(fi.Size()), uint32(fi.Size()>>32) + fmap, err := syscall.CreateFileMapping(syscall.Handle(f.Fd()), nil, syscall.PAGE_READONLY, hi, lo, nil) + if err != nil { + return nil, err + } + defer syscall.CloseHandle(fmap) + + ptr, err := syscall.MapViewOfFile(fmap, syscall.FILE_MAP_READ, 0, 0, uintptr(fi.Size())) + if err != nil { + return nil, err + } + data := (*[1 << 30]byte)(unsafe.Pointer(ptr))[:fi.Size()] + + return data, nil +} + +// Unmap closes the memory-map. +func Unmap(data []byte) error { + if data == nil { + return nil + } + return syscall.UnmapViewOfFile(uintptr(unsafe.Pointer(&data[0]))) +} diff --git a/g/util/gpage/page.go b/g/util/gpage/page.go new file mode 100644 index 000000000..f1c3e58cb --- /dev/null +++ b/g/util/gpage/page.go @@ -0,0 +1,150 @@ +package gpage + +import ( + "math" + "net/http" + "net/url" + "strconv" +) + +type Paginator struct { + Request *http.Request + PerPageNums int + MaxPages int + + nums int64 + pageRange []int + pageNums int + page int +} + +func (p *Paginator) PageNums() int { + if p.pageNums != 0 { + return p.pageNums + } + pageNums := math.Ceil(float64(p.nums) / float64(p.PerPageNums)) + if p.MaxPages > 0 { + pageNums = math.Min(pageNums, float64(p.MaxPages)) + } + p.pageNums = int(pageNums) + return p.pageNums +} + +func (p *Paginator) Nums() int64 { + return p.nums +} + +func (p *Paginator) SetNums(nums interface{}) { + p.nums = int64(nums) +} + +func (p *Paginator) Page() int { + if p.page != 0 { + return p.page + } + if p.Request.Form == nil { + p.Request.ParseForm() + } + p.page, _ = strconv.Atoi(p.Request.Form.Get("p")) + if p.page > p.PageNums() { + p.page = p.PageNums() + } + if p.page <= 0 { + p.page = 1 + } + return p.page +} + +func (p *Paginator) Pages() []int { + if p.pageRange == nil && p.nums > 0 { + var pages []int + pageNums := p.PageNums() + page := p.Page() + switch { + case page >= pageNums-4 && pageNums > 9: + start := pageNums - 9 + 1 + pages = make([]int, 9) + for i, _ := range pages { + pages[i] = start + i + } + case page >= 5 && pageNums > 9: + start := page - 5 + 1 + pages = make([]int, int(math.Min(9, float64(page+4+1)))) + for i, _ := range pages { + pages[i] = start + i + } + default: + pages = make([]int, int(math.Min(9, float64(pageNums)))) + for i, _ := range pages { + pages[i] = i + 1 + } + } + p.pageRange = pages + } + return p.pageRange +} + +func (p *Paginator) PageLink(page int) string { + link, _ := url.ParseRequestURI(p.Request.RequestURI) + values := link.Query() + if page == 1 { + values.Del("p") + } else { + values.Set("p", strconv.Itoa(page)) + } + link.RawQuery = values.Encode() + return link.String() +} + +func (p *Paginator) PageLinkPrev() (link string) { + if p.HasPrev() { + link = p.PageLink(p.Page() - 1) + } + return +} + +func (p *Paginator) PageLinkNext() (link string) { + if p.HasNext() { + link = p.PageLink(p.Page() + 1) + } + return +} + +func (p *Paginator) PageLinkFirst() (link string) { + return p.PageLink(1) +} + +func (p *Paginator) PageLinkLast() (link string) { + return p.PageLink(p.PageNums()) +} + +func (p *Paginator) HasPrev() bool { + return p.Page() > 1 +} + +func (p *Paginator) HasNext() bool { + return p.Page() < p.PageNums() +} + +func (p *Paginator) IsActive(page int) bool { + return p.Page() == page +} + +func (p *Paginator) Offset() int { + return (p.Page() - 1) * p.PerPageNums +} + +func (p *Paginator) HasPages() bool { + return p.PageNums() > 1 +} + +func NewPaginator(req *http.Request, per int, nums interface{}) *Paginator { + p := Paginator{} + p.Request = req + if per <= 0 { + per = 10 + } + p.PerPageNums = per + p.SetNums(nums) + return &p +} \ No newline at end of file diff --git a/g/util/grand/rand.go b/g/util/grand/rand.go new file mode 100644 index 000000000..aecd70f3d --- /dev/null +++ b/g/util/grand/rand.go @@ -0,0 +1,56 @@ +package grand + +import ( + "time" + "math/rand" +) +var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") +var digits = []rune("0123456789") + +// 获得一个 min, max 之间的随机数(min <= x <= max) +func Rand (min, max int) int { + //fmt.Printf("min: %d, max: %d\n", min, max) + if min >= max { + return min + } + rand.Seed(time.Now().UnixNano()) + n := rand.Intn(max + 1) + if n < min { + return Rand(min, max) + } + return n +} + +// 获得指定长度的随机字符串(可能包含数字和字母) +func RandStr(n int) string { + rand.Seed(time.Now().UnixNano()) + b := make([]rune, n) + for i := range b { + if rand.Intn(2) == 1 { + b[i] = digits[rand.Intn(10)] + } else { + b[i] = letters[rand.Intn(52)] + } + } + return string(b) +} + +// 获得指定长度的随机数字字符串 +func RandDigits(n int) string { + rand.Seed(time.Now().UnixNano()) + b := make([]rune, n) + for i := range b { + b[i] = digits[rand.Intn(10)] + } + return string(b) +} + +// 获得指定长度的随机字母字符串 +func RandLetters(n int) string { + rand.Seed(time.Now().UnixNano()) + b := make([]rune, n) + for i := range b { + b[i] = letters[rand.Intn(52)] + } + return string(b) +} diff --git a/g/util/gregx/regx.go b/g/util/gregx/regx.go new file mode 100644 index 000000000..1189931d6 --- /dev/null +++ b/g/util/gregx/regx.go @@ -0,0 +1,14 @@ +package gregx + +import ( + "regexp" +) + +// 正则表达式是否匹配 +func IsMatch(val, pattern string) bool { + match, err := regexp.Match(pattern, []byte(val)) + if err != nil { + return false + } + return match +} \ No newline at end of file diff --git a/g/util/gtime/time.go b/g/util/gtime/time.go new file mode 100644 index 000000000..2342df977 --- /dev/null +++ b/g/util/gtime/time.go @@ -0,0 +1,57 @@ +package gtime + +import ( + "time" +) + +// 类似与js中的SetTimeout，一段时间后执行回调函数 +func SetTimeout(t time.Duration, callback func()) { + go func() { + time.Sleep(t) + callback() + }() +} + +// 类似与js中的SetInterval，每隔一段时间后执行回调函数，当回调函数返回true，那么继续执行，否则终止执行，该方法是异步的 +// 注意：由于采用的是循环而不是递归操作，因此间隔时间将会以上一次回调函数执行完成的时间来计算 +func SetInterval(t time.Duration, callback func() bool) { + go func() { + for { + time.Sleep(t) + r := callback() + if !r { + break; + } + } + }() +} + +// 获取当前的纳秒数 +func Nanosecond() int64 { + return time.Now().UnixNano() +} + +// 获取当前的微秒数 +func Microsecond() int64 { + return time.Now().UnixNano()/1e3 +} + +// 获取当前的毫秒数 +func Millisecond() int64 { + return time.Now().UnixNano()/1e6 +} + +// 获取当前的秒数(时间戳) +func Second() int64 { + return time.Now().UnixNano()/1e9 +} + +// 获得当前的日期(例如：2006-01-02) +func Date() string { + return time.Now().Format("2006-01-02") +} + +// 获得当前的时间(例如：2006-01-02 15:04:05) +func Datetime() string { + return time.Now().Format("2006-01-02 15:04:05") +} diff --git a/g/util/gutil/util.go b/g/util/gutil/util.go new file mode 100644 index 000000000..3a1565b58 --- /dev/null +++ b/g/util/gutil/util.go @@ -0,0 +1,19 @@ +package gutil + +// 便利数组查找字符串索引位置，如果不存在则返回-1 +func StringSearch (a []string, s string) int { + for i, v := range a { + if s == v { + return i + } + } + return -1 +} + +// 判断字符串是否在数组中 +func StringInArray (a []string, s string) bool { + return StringSearch(a, s) != -1 +} + + + diff --git a/gexample/db/kvdb/gkvdb_analysis.go b/gexample/db/kvdb/gkvdb_analysis.go new file mode 100644 index 000000000..248f2a247 --- /dev/null +++ b/gexample/db/kvdb/gkvdb_analysis.go @@ -0,0 +1,63 @@ +package main + +import ( + "fmt" + "g/os/gfile" + "g/encoding/gbinary" +) + +type Block struct { + index int + size uint +} + +func main() { + + content := gfile.GetBinContents("/tmp/blocks") + + blocks := make([]Block, 0) + for i := 0; i < len(content); i += 12 { + block := Block{ + int(gbinary.DecodeToInt64(content[i : i + 8])), + uint(gbinary.DecodeToUint32(content[i + 8 : i + 12])), + } + blocks = append(blocks, block) + } + for i := 0; i < len(blocks); i++ { + if i + 1 == len(blocks) { + break + } + //fmt.Println(blocks[i].index, blocks[i].size) + if blocks[i].index + int(blocks[i].size) >= blocks[i+1].index { + fmt.Println(blocks[i].index, "+", blocks[i].size, ">=", blocks[i+1].index) + break + } + } + + + //fs := gfilespace.New() + //blocks := make([]gfilespace.Block, 0) + //for i := 0; i < len(content); i += 12 { + // fs.AddBlock( + // int(gbinary.DecodeToInt64(content[i : i + 8])), + // uint(gbinary.DecodeToUint32(content[i + 8 : i + 12])), + // ) + // + //} + //for _, v := range fs.GetAllBlocks() { + // blocks = append(blocks, v) + //} + // + //for i := 0; i < len(blocks); i++ { + // if i + 1 == len(blocks) { + // break + // } + // fmt.Println(blocks[i].Index(), blocks[i].Size()) + // if blocks[i].Index() + int(blocks[i].Size()) >= blocks[i+1].Index() { + // fmt.Println(blocks[i].Index(), "+", blocks[i].Size(), ">=", blocks[i+1].Index()) + // break + // } + //} + + //fmt.Println(blocks) +} \ No newline at end of file diff --git a/gexample/db/mysql/mysql.go b/gexample/db/mysql/mysql.go new file mode 100644 index 000000000..3cbcaeebf --- /dev/null +++ b/gexample/db/mysql/mysql.go @@ -0,0 +1,382 @@ +package main + +import ( + "fmt" + "time" + "strconv" + "g/database/gdb" +) + +// 本文件用于gf框架的mysql数据库操作示例，不作为单元测试使用 + +var db gdb.Link + +// 初始化配置及创建数据库 +func init () { + gdb.AddDefaultConfigNode(gdb.ConfigNode { + Host : "127.0.0.1", + Port : 3306, + User : "root", + Pass : "123456", + Name : "test2", + Type : "mysql", + Role : "master", + Charset : "utf8", + }) + db, _ = gdb.Instance() + + //gdb.SetConfig(gdb.ConfigNode { + // Host : "127.0.0.1", + // Port : 3306, + // User : "root", + // Pass : "123456", + // Name : "test", + // Type : "mysql", + //}) + //db, _ = gdb.Instance() + + //gdb.SetConfig(gdb.Config { + // "default" : gdb.ConfigGroup { + // gdb.ConfigNode { + // Host : "127.0.0.1", + // Port : "3306", + // User : "root", + // Pass : "123456", + // Name : "test", + // Type : "mysql", + // Role : "master", + // Priority : 100, + // }, + // gdb.ConfigNode { + // Host : "127.0.0.2", + // Port : "3306", + // User : "root", + // Pass : "123456", + // Name : "test", + // Type : "mysql", + // Role : "master", + // Priority : 100, + // }, + // gdb.ConfigNode { + // Host : "127.0.0.3", + // Port : "3306", + // User : "root", + // Pass : "123456", + // Name : "test", + // Type : "mysql", + // Role : "master", + // Priority : 100, + // }, + // gdb.ConfigNode { + // Host : "127.0.0.4", + // Port : "3306", + // User : "root", + // Pass : "123456", + // Name : "test", + // Type : "mysql", + // Role : "master", + // Priority : 100, + // }, + // }, + //}) + //db, _ = gdb.Instance() +} + + + +// 创建测试数据库 +func create() { + fmt.Println("create:") + _, err := db.Exec("CREATE DATABASE IF NOT EXISTS test") + if (err != nil) { + fmt.Println(err) + } + + s := ` + CREATE TABLE IF NOT EXISTS user ( + uid INT(10) UNSIGNED AUTO_INCREMENT, + name VARCHAR(45), + PRIMARY KEY (uid) + ) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8 + ` + _, err = db.Exec(s) + if (err != nil) { + fmt.Println(err) + } + + s = ` + CREATE TABLE IF NOT EXISTS user_detail ( + uid INT(10) UNSIGNED AUTO_INCREMENT, + site VARCHAR(255), + PRIMARY KEY (uid) + ) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8 + ` + + _, err = db.Exec(s) + if (err != nil) { + fmt.Println(err) + } + fmt.Println() +} + +// 数据写入 +func insert() { + fmt.Println("insert:") + r, err := db.Insert("user", &gdb.Map { + "name": "john", + }) + if (err == nil) { + uid, err2 := r.LastInsertId() + if err2 == nil { + r, err = db.Insert("user_detail", &gdb.Map { + "uid" : uid, + "site" : "http://johng.cn", + }) + if err == nil { + fmt.Printf("uid: %d\n", uid) + } else { + fmt.Println(err) + } + } else { + fmt.Println(err2) + } + } else { + fmt.Println(err) + } + fmt.Println() +} + + +// 基本sql查询 +func query() { + fmt.Println("query:") + list, err := db.GetAll("select * from user limit 2") + if err == nil { + fmt.Println(list) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// replace into +func replace() { + fmt.Println("replace:") + r, err := db.Save("user", &gdb.Map { + "uid" : 1, + "name" : "john", + }) + if (err == nil) { + fmt.Println(r.LastInsertId()) + fmt.Println(r.RowsAffected()) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// 数据保存 +func save() { + fmt.Println("save:") + r, err := db.Save("user", &gdb.Map { + "uid" : 1, + "name" : "john", + }) + if (err == nil) { + fmt.Println(r.LastInsertId()) + fmt.Println(r.RowsAffected()) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// 批量写入 +func batchInsert() { + fmt.Println("batchInsert:") + err := db.BatchInsert("user", &gdb.List { + {"name": "john_" + strconv.FormatInt(time.Now().UnixNano(), 10)}, + {"name": "john_" + strconv.FormatInt(time.Now().UnixNano(), 10)}, + {"name": "john_" + strconv.FormatInt(time.Now().UnixNano(), 10)}, + {"name": "john_" + strconv.FormatInt(time.Now().UnixNano(), 10)}, + }, 10) + if err != nil { + fmt.Println(err) + } + fmt.Println() +} + +// 数据更新 +func update1() { + fmt.Println("update1:") + r, err := db.Update("user", &gdb.Map {"name": "john1"}, "uid=?", 1) + if (err == nil) { + fmt.Println(r.LastInsertId()) + fmt.Println(r.RowsAffected()) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// 数据更新 +func update2() { + fmt.Println("update2:") + r, err := db.Update("user", "name='john2'", "uid=1") + if (err == nil) { + fmt.Println(r.LastInsertId()) + fmt.Println(r.RowsAffected()) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// 数据更新 +func update3() { + fmt.Println("update3:") + r, err := db.Update("user", "name=?", "uid=?", "john2", 1) + if (err == nil) { + fmt.Println(r.LastInsertId()) + fmt.Println(r.RowsAffected()) + } else { + fmt.Println(err) + } + fmt.Println() +} + + +// 链式查询操作1 +func linkopSelect1() { + fmt.Println("linkopSelect1:") + r, err := db.Table("user u").LeftJoin("user_detail ud", "u.uid=ud.uid").Fields("u.*, ud.site").Condition("u.uid > ?", 1).Limit(0, 2).Select() + if (err == nil) { + fmt.Println(r) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// 链式查询操作2 +func linkopSelect2() { + fmt.Println("linkopSelect2:") + r, err := db.Table("user u").LeftJoin("user_detail ud", "u.uid=ud.uid").Fields("u.*,ud.site").Condition("u.uid=?", 1).One() + if (err == nil) { + fmt.Println(r) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// 链式查询操作3 +func linkopSelect3() { + fmt.Println("linkopSelect3:") + r, err := db.Table("user u").LeftJoin("user_detail ud", "u.uid=ud.uid").Fields("ud.site").Condition("u.uid=?", 1).Value() + if (err == nil) { + fmt.Println(r.(string)) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// 错误操作 +func linkopUpdate1() { + fmt.Println("linkopUpdate1:") + r, err := db.Table("henghe_setting").Update() + if (err == nil) { + fmt.Println(r.RowsAffected()) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// 通过Map指针方式传参方式 +func linkopUpdate2() { + fmt.Println("linkopUpdate2:") + r, err := db.Table("user").Data(&gdb.Map{"name" : "john2"}).Condition("name=?", "john").Update() + if (err == nil) { + fmt.Println(r.RowsAffected()) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// 通过字符串方式传参 +func linkopUpdate3() { + fmt.Println("linkopUpdate3:") + r, err := db.Table("user").Data("name='john3'").Condition("name=?", "john2").Update() + if (err == nil) { + fmt.Println(r.RowsAffected()) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// 主从io复用测试，在mysql中使用 show full processlist 查看链接信息 +func keepPing() { + fmt.Println("keepPing:") + for { + fmt.Println("ping...") + err := db.PingMaster() + if err != nil { + fmt.Println(err) + return + } + err = db.PingSlave() + if err != nil { + fmt.Println(err) + return + } + time.Sleep(1*time.Second) + } +} + +// 数据库单例测试，在mysql中使用 show full processlist 查看链接信息 +func instance() { + fmt.Println("instance:") + db1, _ := gdb.Instance() + db2, _ := gdb.Instance() + db3, _ := gdb.Instance() + for { + fmt.Println("ping...") + db1.PingMaster() + db1.PingSlave() + db2.PingMaster() + db2.PingSlave() + db3.PingMaster() + db3.PingSlave() + time.Sleep(1*time.Second) + } +} + + +func main() { + + //create() + //create() + //insert() + //query() + //replace() + //save() + //batchInsert() + //update1() + //update2() + //update3() + //linkopSelect1() + //linkopSelect2() + //linkopSelect3() + //linkopUpdate1() + //linkopUpdate2() + //linkopUpdate3() + keepPing() +} \ No newline at end of file diff --git a/gexample/db/pgsql/pgsql.go b/gexample/db/pgsql/pgsql.go new file mode 100644 index 000000000..6e5208f51 --- /dev/null +++ b/gexample/db/pgsql/pgsql.go @@ -0,0 +1,284 @@ +package main + +import ( + "fmt" + "time" + "strconv" + "g/database/gdb" +) + +// 本文件用于gf框架的postgresql数据库操作示例，不作为单元测试使用 + +var db gdb.Link + +func init () { + gdb.AddDefaultConfigNode(gdb.ConfigNode { + Host : "127.0.0.1", + Port : 5432, + User : "postgres", + Pass : "123456", + Name : "test", + Type : "pgsql", + }) + db, _ = gdb.Instance() +} + + + +// 创建测试数据库 +func create() { + fmt.Println("create:") + _, err := db.Exec("CREATE SCHEMA IF NOT EXISTS \"test\"") + if (err != nil) { + fmt.Println(err) + } + + s := `CREATE TABLE IF NOT EXISTS "user" ( + uid int PRIMARY KEY, + name TEXT NOT NULL + ) + ` + _, err = db.Exec(s) + if (err != nil) { + fmt.Println(err) + } + + s = ` + CREATE TABLE IF NOT EXISTS user_detail ( + uid int PRIMARY KEY, + site TEXT NOT NULL + ) + ` + _, err = db.Exec(s) + if (err != nil) { + fmt.Println(err) + } + fmt.Println() +} + +// 数据写入 +func insert() { + fmt.Println("insert:") + r, err := db.Insert("user", &gdb.Map { + "uid" : 1, + "name": "john", + }) + if (err == nil) { + uid, err2 := r.LastInsertId() + if err2 == nil { + r, err = db.Insert("user_detail", &gdb.Map { + "uid" : string(uid), + "site" : "http://johng.cn", + }) + if err == nil { + fmt.Printf("uid: %d\n", uid) + } else { + fmt.Println(err) + } + } else { + fmt.Println(err2) + } + } else { + fmt.Println(err) + } + fmt.Println() +} + + +// 基本sql查询 +func query() { + fmt.Println("query:") + list, err := db.GetAll("select * from \"user\"") + if err == nil { + fmt.Println(list) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// replace into +func replace() { + fmt.Println("replace:") + r, err := db.Save("user", &gdb.Map { + "uid": "1", + "name": "john", + }) + if (err == nil) { + fmt.Println(r.LastInsertId()) + fmt.Println(r.RowsAffected()) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// 数据保存 +func save() { + fmt.Println("save:") + r, err := db.Save("user", &gdb.Map { + "uid" : "1", + "name" : "john", + }) + if (err == nil) { + fmt.Println(r.LastInsertId()) + fmt.Println(r.RowsAffected()) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// 批量写入 +func batchInsert() { + fmt.Println("batchInsert:") + err := db.BatchInsert("user", &gdb.List { + {"name": "john_" + strconv.FormatInt(time.Now().UnixNano(), 10)}, + {"name": "john_" + strconv.FormatInt(time.Now().UnixNano(), 10)}, + {"name": "john_" + strconv.FormatInt(time.Now().UnixNano(), 10)}, + {"name": "john_" + strconv.FormatInt(time.Now().UnixNano(), 10)}, + }, 10) + if err != nil { + fmt.Println(err) + } + fmt.Println() +} + +// 数据更新 +func update1() { + fmt.Println("update1:") + r, err := db.Update("user", &gdb.Map {"name": "john1"}, "uid=?", 1) + if (err == nil) { + fmt.Println(r.LastInsertId()) + fmt.Println(r.RowsAffected()) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// 数据更新 +func update2() { + fmt.Println("update2:") + r, err := db.Update("user", "name='john2'", "uid=1") + if (err == nil) { + fmt.Println(r.LastInsertId()) + fmt.Println(r.RowsAffected()) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// 数据更新 +func update3() { + fmt.Println("update3:") + r, err := db.Update("user", "name=?", "uid=?", "john2", 1) + if (err == nil) { + fmt.Println(r.LastInsertId()) + fmt.Println(r.RowsAffected()) + } else { + fmt.Println(err) + } + fmt.Println() +} + + +// 链式查询操作 +func linkopSelect() { + fmt.Println("linkopSelect:") + r, err := db.Table("user u"). + LeftJoin("user_detail ud", "u.uid=ud.uid"). + Fields("u.*, ud.site"). + Condition("u.uid > ?", 1). + Limit(0, 2).Select() + if (err == nil) { + fmt.Println(r) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// 错误操作 +func linkopUpdate1() { + fmt.Println("linkopUpdate1:") + r, err := db.Table("henghe_setting").Update() + if (err == nil) { + fmt.Println(r.RowsAffected()) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// 通过Map指针方式传参方式 +func linkopUpdate2() { + fmt.Println("linkopUpdate2:") + r, err := db.Table("user").Data(&gdb.Map{"name" : "john2"}).Condition("name=?", "john").Update() + if (err == nil) { + fmt.Println(r.RowsAffected()) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// 通过字符串方式传参 +func linkopUpdate3() { + fmt.Println("linkopUpdate3:") + r, err := db.Table("user").Data("name='john3'").Condition("name=?", "john2").Update() + if (err == nil) { + fmt.Println(r.RowsAffected()) + } else { + fmt.Println(err) + } + fmt.Println() +} + +// 主从io复用测试，在mysql中使用 show full processlist 查看链接信息 +func keepPing() { + fmt.Println("keepPing:") + for { + fmt.Println("ping...") + db.PingMaster() + db.PingSlave() + time.Sleep(1*time.Second) + } +} + +// 数据库单例测试，在mysql中使用 show full processlist 查看链接信息 +func instance() { + fmt.Println("instance:") + db1, _ := gdb.Instance() + db2, _ := gdb.Instance() + db3, _ := gdb.Instance() + for { + fmt.Println("ping...") + db1.PingMaster() + db1.PingSlave() + db2.PingMaster() + db2.PingSlave() + db3.PingMaster() + db3.PingSlave() + time.Sleep(1*time.Second) + } +} + + +func main() { + create() + create() + insert() + query() + replace() + save() + batchInsert() + update1() + update2() + update3() + linkopSelect() + linkopUpdate1() + linkopUpdate2() + linkopUpdate3() +} \ No newline at end of file diff --git a/gexample/encoding/ghash.go b/gexample/encoding/ghash.go new file mode 100644 index 000000000..fe873b5a9 --- /dev/null +++ b/gexample/encoding/ghash.go @@ -0,0 +1,20 @@ +package main + +import ( + "strconv" + "fmt" + "g/encoding/ghash" +) + +func main () { + m := make(map[uint64]bool) + for i := 0; i < 100000000; i++ { + hash := ghash.BKDRHash64([]byte("key_" + strconv.Itoa(i))) + if _, ok := m[hash]; ok { + fmt.Printf("duplicated hash %d\n", hash) + } else { + m[hash] = true + } + } +} + diff --git a/gexample/encoding/json.go b/gexample/encoding/json.go new file mode 100644 index 000000000..a122e44b2 --- /dev/null +++ b/gexample/encoding/json.go @@ -0,0 +1,112 @@ +package main + +import ( + "fmt" + //"encoding/json" + "g/encoding/gjson" +) + +type City struct { + Age string + CityId int + CityName string + ProvinceId int + //CityOrder int +} + +func main() { + //data := `[{"CityId":1, "CityName":"北京", "ProvinceId":1, "CityOrder":1}, {"CityId":5, "CityName":"成都", "ProvinceId":27, "CityOrder":1}]` + data := `{"name":"中国","age":31,"list":[["a","b","c"],["d","e","f"]],"items":{"title":"make\"he moon","name":"make'he moon","content":"'[}]{[}he moon"}}` + //data := `[{"CityId":18,"CityName":"西安","ProvinceId":27,"CityOrder":1},{"CityId":53,"CityName":"广州","ProvinceId":27,"CityOrder":1}]` + //data := `{"name" : "中国", "age" : 31, "items":[1,2,3]}` + //data := `[["a","b","c"],["d","e","f"]]` + //data := `["a","b","c"]` + //json := ` + //[1,{"a":2}, + //{"a":{}}, + //{"a":[]}, + //{"a":[{}]}, + //{"{[a" : "\"2,:3," a ":33}]"}]` // 错误的json + //data := `["a","b","c"` // 错误的json + //data := `,{ "name" : "中国", "age" : 31, "items":[1,2]:}` //错误的json + + v := gjson.DecodeToJson(&data) + fmt.Println(v.GetNumber("list")) + + //v := map[string]interface{} { + // + // "name" : "中国", + // "age" : 11, + // "list" : []interface{} { + // 1,2,3,4, + // }, + //} + //r, _ := json.MarshalIndent(v, "", "\t") + //fmt.Println(string(r)) + //s, _ := gjson.Encode(v) + //fmt.Println(*s) + + + //p, err := gjson.Decode(&data) + //if err == nil { + // //p.Print() + // //fmt.Println(p.Get("0")) + // fmt.Println(p.GetMap("0")) + //} else { + // fmt.Println(err) + //} + //fmt.Println() + //fmt.Println() + ////v := make(map[string]interface{}) + ////i := 31 + ////j := "john" + ////v["age"] = i + ////v["name"] = make(map[string]interface{}) + ////t := v["name"] + ////t.(map[string]interface{})["n"] = j + //// + ////fmt.Println(v) + //var s struct{ + // v interface{} + // p interface{} + //} + //v := make(map[string]interface{}) + //s.v = v + //s.p = &v + //c := (*s.p.(*map[string]interface{})) + //c["name1"] = "john1" + // + //t := make(map[string]interface{}) + //c["/"] = t + //s.p = &t + //t["name2"] = "john2" + // + //c2 := (*s.p.(*map[string]interface{})) + //c2["name3"] = "john3" + // + ////t2[2] = 100 + //fmt.Println(s.v) + + + //a := map[string]interface{} { + // "name" : "john", + // "list" : []interface{}{ + // 1,2,3, "fuck", + // }, + // "item" : map[string]string { + // "n1" : "v1", + // "n2" : "v2", + // "n3" : "v3", + // }, + //} + //fmt.Println(json.M) + + // + //var a = []int{1,2,3} + //var b = []int{4,5,6, 7,8} + //cc := make([]int, len(a) + 12) + //a = cc + //copy(a, b) + //fmt.Println(a) + //fmt.Println(b) +} \ No newline at end of file diff --git a/gexample/encoding/json_test.go b/gexample/encoding/json_test.go new file mode 100644 index 000000000..f7108c53c --- /dev/null +++ b/gexample/encoding/json_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "testing" + "g/encoding/gjson" + "encoding/json" + "log" +) + +// go test json_test.go -bench=".*" + +var data = `[{"CityId":1, "CityName":"北京", "ProvinceId":1, "CityOrder":1}, {"CityId":5, "CityName":"成都", "ProvinceId":27, "CityOrder":1}]` + +func BenchmarkJsonDecode(b *testing.B) { + b.N = 1000000 + for i := 0; i < b.N; i++ { + gjson.Decode(data) + } +} + +func BenchmarkJsonDecodeByUnmarshal(b *testing.B) { + b.N = 1000000 + for i := 0; i < b.N; i++ { + var citys interface{} + if err := json.Unmarshal([]byte(data), &citys); err != nil { + log.Fatalf("JSON unmarshaling failed: %s", err) + } + //fmt.Println(citys) + } +} + diff --git a/gexample/net/gluster_client.go b/gexample/net/gluster_client.go new file mode 100644 index 000000000..a06df4385 --- /dev/null +++ b/gexample/net/gluster_client.go @@ -0,0 +1,156 @@ +package main + +import ( + "fmt" + "g/net/ghttp" +) + +var kvUrl string = "http://192.168.2.102:4168/kv" +var nodeUrl string = "http://192.168.2.102:4168/node" +var serviceUrl string = "http://192.168.2.102:4168/service" + +// kv操作 +func addKV() { + c := ghttp.NewClient() + r := c.Put(kvUrl, "{\"name1\":\"john1\", \"name2\":\"john2\"}") + fmt.Println("addKV:", r.ReadAll()) +} + +func getAllKV() { + c := ghttp.NewClient() + r := c.Get(kvUrl) + fmt.Println("getAllKV:", r.ReadAll()) +} + +func getOneKV() { + c := ghttp.NewClient() + r := c.Get(kvUrl + "?k=name1") + fmt.Println("getOneKV:", r.ReadAll()) +} + +func editKV() { + c := ghttp.NewClient() + r := c.Post(kvUrl, "{\"name1\":\"john3\", \"name2\":\"john4\"}") + fmt.Println("editKV:", r.ReadAll()) +} + +func removeKV() { + c := ghttp.NewClient() + r := c.Delete(kvUrl, "[\"name1\"]") + fmt.Println("removeKV:", r.ReadAll()) +} + + +// node操作 +func addNode() { + c := ghttp.NewClient() + r := c.Put(nodeUrl, "[\"172.17.42.1\"]") + fmt.Println("addNode:", r.ReadAll()) +} + +func getAllNode() { + c := ghttp.NewClient() + r := c.Get(nodeUrl) + fmt.Println("getAllNode:", r.ReadAll()) +} + +func removeNode() { + c := ghttp.NewClient() + r := c.Delete(nodeUrl, "[\"172.17.42.1\"]") + fmt.Println("removeNode:", r.ReadAll()) +} + + +// service操作 +func getAllService() { + c := ghttp.NewClient() + r := c.Get(serviceUrl) + fmt.Println("getAllService:", r.ReadAll()) +} + +func getOneService() { + c := ghttp.NewClient() + r := c.Get(serviceUrl + "?name=Site Database") + fmt.Println("getOneService:", r.ReadAll()) +} + +func addDatabaseService() { + c := ghttp.NewClient() + s := ` +{ + "name" : "Site Database", + "type" : "mysql", + "list" : [ + {"host":"192.168.2.102", "port":"3306", "user":"root", "pass":"123456", "database":"test"}, + {"host":"192.168.2.124", "port":"3306", "user":"root", "pass":"123456", "database":"tongwujie"} + ] +} + ` + r := c.Put(serviceUrl, s) + fmt.Println("addDatabaseService:", r.ReadAll()) +} + +func editDatabaseService() { + c := ghttp.NewClient() + s := ` +{ + "name" : "Site Database2", + "type" : "mysql", + "list" : [ + {"host":"192.168.2.102", "port":"3306", "user":"root", "pass":"123456", "database":"test"}, + {"host":"192.168.2.124", "port":"3306", "user":"root", "pass":"123456", "database":"tongwujie"} + ] +} + ` + r := c.Post(serviceUrl, s) + fmt.Println("editDatabaseService:", r.ReadAll()) +} + +func removeDatabaseService() { + c := ghttp.NewClient() + r := c.Delete(serviceUrl, "[\"Site Database2\"]") + fmt.Println("removeDatabaseService:", r.ReadAll()) +} + + +func addWebService() { + c := ghttp.NewClient() + s := ` +{ + "name" : "Site", + "type" : "web", + "list" : [ + {"url":"http://baidu.com", "check":"http://itsadeadlink.com"}, + {"url":"http://baidu.com"} + ] +} + ` + r := c.Put(serviceUrl, s) + fmt.Println("addWebService:", r.ReadAll()) +} + +func editWebService() { + c := ghttp.NewClient() + s := ` +{ + "name" : "Site2", + "type" : "web", + "list" : [ + {"url":"http://baidu.com"}, + {"url":"http://baidu.com"} + ] +} + ` + r := c.Post(serviceUrl, s) + fmt.Println("editWebService:", r.ReadAll()) +} + +func removeWebService() { + c := ghttp.NewClient() + r := c.Delete(serviceUrl, "[\"Site2\"]") + fmt.Println("removeWebService:", r.ReadAll()) +} + +func main() { + addWebService() +} diff --git a/gexample/net/http_client.go b/gexample/net/http_client.go new file mode 100644 index 000000000..0c538c0aa --- /dev/null +++ b/gexample/net/http_client.go @@ -0,0 +1,14 @@ +package main + +import ( + "fmt" + "g/net/ghttp" +) + + +func main() { + c := ghttp.NewClient() + r := c.Get("http://192.168.2.124") + + fmt.Println(r.StatusCode) +} diff --git a/gexample/net/http_server.go b/gexample/net/http_server.go new file mode 100644 index 000000000..318732c76 --- /dev/null +++ b/gexample/net/http_server.go @@ -0,0 +1,29 @@ +package main + +import ( + "net/http" + "io" + "g/net/ghttp" +) + +func HelloServer1(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "hello1!\n") +} +func HelloServer2(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "hello2\n") +} +func main() { + s := ghttp.New() + s.SetAddr(":8199") + s.SetIndexFolder(true) + s.SetServerRoot("/home/john/Workspace/") + s.BindHandleByMap(ghttp.HandlerMap { + "/h": HelloServer1, + "/h1": HelloServer1, + "/h2": HelloServer1, + "/h3": HelloServer1, + }) + s.BindHandle("/hello1", HelloServer1) + s.BindHandle("/hello2", HelloServer2) + s.Run() +} diff --git a/gexample/net/raft.go b/gexample/net/raft.go new file mode 100644 index 000000000..a914f0c67 --- /dev/null +++ b/gexample/net/raft.go @@ -0,0 +1,25 @@ +package main + +import ( + "g/net/graft" + "g/net/gip" + "log" +) + + + +func main() { + ips, err := gip.IntranetIP() + if err != nil { + log.Println(err) + return + } + + for _, ip := range ips { + //fmt.Println(ip) + graft.NewServerByIp(ip).Run() + } + select { + + } +} \ No newline at end of file diff --git a/gexample/net/raft_client.go b/gexample/net/raft_client.go new file mode 100644 index 000000000..193c337d5 --- /dev/null +++ b/gexample/net/raft_client.go @@ -0,0 +1,29 @@ +package main + +import ( + "net" + "log" + "g/net/graft" + "fmt" + "g/encoding/gjson" +) + +func rpcLogSet() { + conn, err := net.Dial("tcp", "192.168.2.124:4167") + if err != nil { + log.Println(err) + return + } + + entry := graft.LogRequest{} + entry.Key = "name3" + entry.Value = "john3" + fmt.Println(*gjson.Encode(entry)) + e := graft.SendMsg(conn, 100, *gjson.Encode(entry)) + fmt.Println(e) + conn.Close() +} + +func main() { + rpcLogSet() +} \ No newline at end of file diff --git a/gexample/net/scanner.go b/gexample/net/scanner.go new file mode 100644 index 000000000..c2910ec4c --- /dev/null +++ b/gexample/net/scanner.go @@ -0,0 +1,23 @@ +package main + +import ( + "g/net/gscanner" + "net" + "fmt" + "time" +) + +func main() { + //gscanner.New().SetTimeout(3*time.Second).ScanIp("192.168.2.1", "192.168.2.255", 80, func(conn net.Conn){ + // fmt.Println(conn.RemoteAddr()) + //}) + + gscanner.New().SetTimeout(3*time.Second).ScanIp("120.76.249.1", "120.76.249.255", 80, func(conn net.Conn){ + fmt.Println(conn.RemoteAddr()) + }) + + //gscanner.New().SetTimeout(6*time.Second).ScanPort("120.76.249.2", func(conn net.Conn){ + // //fmt.Println("yes") + // fmt.Println(conn.RemoteAddr()) + //}) +} \ No newline at end of file diff --git a/gexample/net/tcp_server.go b/gexample/net/tcp_server.go new file mode 100644 index 000000000..3f0a1ace9 --- /dev/null +++ b/gexample/net/tcp_server.go @@ -0,0 +1,47 @@ +package main + +import ( + "net" + "fmt" + "g/net/gtcp" + "io" + "log" + "time" + "g/util/gutil" +) + +func main() { + gtcp.NewServer(":8999", func(conn net.Conn) { + + + try := 0 + buffersize := 5 + data := make([]byte, 0) + for { + buffer := make([]byte, buffersize) + length, err := conn.Read(buffer) + if err != nil { + log.Println(err) + if err != io.EOF { + log.Println("node recieve:", err, "try:", try) + } + if try > 2 { + break; + } + try ++ + time.Sleep(100 * time.Millisecond) + } else { + if length == buffersize { + data = gutil.MergeSlice(data, buffer) + } else { + data = gutil.MergeSlice(data, buffer[0:length]) + break; + } + } + } + fmt.Println(string(data)) + }).Run() + select { + + } +} \ No newline at end of file diff --git a/gexample/net/udp.go b/gexample/net/udp.go new file mode 100644 index 000000000..da0d30c4f --- /dev/null +++ b/gexample/net/udp.go @@ -0,0 +1,21 @@ +package main + +import ( + "net" + "fmt" + "os" +) + +func main() { + conn, err := net.Dial("udp", "127.0.0.1:8999") + defer conn.Close() + if err != nil { + os.Exit(1) + } + + conn.Write([]byte("")) + var msg [20]byte + n, err := conn.Read(msg[0:]) + + fmt.Println(string(msg[0:n])) +} \ No newline at end of file diff --git a/gexample/net/udp_server.go b/gexample/net/udp_server.go new file mode 100644 index 000000000..52ee4b029 --- /dev/null +++ b/gexample/net/udp_server.go @@ -0,0 +1,19 @@ +package main + +import ( + "net" + "fmt" + "g/net/gudp" +) + +func main() { + gudp.NewServer(":8999", func(conn *net.UDPConn) { + var buf [1024]byte + count, raddr, err := conn.ReadFromUDP(buf[0:]) + if err != nil { + return + } + fmt.Println(raddr.String() + ":", string(buf[0:count])) + _, err = conn.WriteToUDP([]byte("hi"), raddr) + }).Run() +} \ No newline at end of file diff --git a/gexample/os/console.go b/gexample/os/console.go new file mode 100644 index 000000000..7a315ec9e --- /dev/null +++ b/gexample/os/console.go @@ -0,0 +1,21 @@ +package main + +import ( + "fmt" + "g/os/gconsole" +) + +func doEcho() { + fmt.Println("do echo") +} + +func main() { + fmt.Println(gconsole.Value.GetAll()) + + fmt.Println(gconsole.Value.GetIndex(1)) + + gconsole.BindHandle("echo", doEcho) + gconsole.RunHandle("echo") + + gconsole.AutoRun() +} diff --git a/gexample/os/file.go b/gexample/os/file.go new file mode 100644 index 000000000..d58c5e0f9 --- /dev/null +++ b/gexample/os/file.go @@ -0,0 +1,47 @@ +package main + +import ( + "g/os/gfile" + "fmt" +) + +var dirpath1 = "/home/john/Workspace/temp/" +var dirpath2 = "/home/john/Workspace/temp/1" +var filepath1 = "/home/john/Workspace/temp/test.php" +var filepath2 = "/tmp/tmp.test" + + +type BinData struct{ + name string + age int +} + +func info () { + fmt.Println(gfile.Info(dirpath1)) +} + +func scanDir() { + files := gfile.ScanDir(dirpath1) + fmt.Println(files) +} + +func getContents() { + fmt.Printf("%s\n", gfile.GetContents(filepath1)) +} + +func putContents() { + fmt.Println(gfile.PutContentsAppend(filepath2, []byte("123"))) +} + +func putBinContents() { + data := []byte(BinData{"john", 31}) + fmt.Println(gfile.PutContents(filepath2, data)) +} + +func main() { + //info() + //getContents() + //putContents() + putBinContents() + //scanDir() +} \ No newline at end of file diff --git a/gexample/os/gcache_test.go b/gexample/os/gcache_test.go new file mode 100644 index 000000000..59d8d9929 --- /dev/null +++ b/gexample/os/gcache_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "testing" + "g/os/gcache" +) + +var cache *gcache.Cache = gcache.New() + +func BenchmarkSet(b *testing.B) { + b.N = 1000000 + for i := 0; i < 1000000; i ++ { + cache.Set(string(i), i, 0) + } +} + +func BenchmarkSetWithExpire(b *testing.B) { + b.N = 1000000 + for i := 0; i < 1000000; i ++ { + cache.Set(string(i), i, 60) + } +} + +func BenchmarkGet1(b *testing.B) { + b.N = 1000000 + for i := 0; i < 1000000; i ++ { + cache.Get(string(i)) + } +} + +func BenchmarkGet2(b *testing.B) { + b.N = 1000000 + for i := 0; i < 1000000; i ++ { + cache.Get(string(i)) + } +} + +func BenchmarkRemove(b *testing.B) { + b.N = 1000000 + for i := 0; i < 1000000; i ++ { + cache.Remove(string(i)) + } +} \ No newline at end of file diff --git a/gexample/os/gfilespace.go b/gexample/os/gfilespace.go new file mode 100644 index 000000000..0d90e6b4a --- /dev/null +++ b/gexample/os/gfilespace.go @@ -0,0 +1,70 @@ +package main + +import ( + "g/os/gfilespace" + "fmt" +) + + + +func main() { + + //t1 := gtime.Microsecond() + space := gfilespace.New() + + space.AddBlock(0, 10) + space.AddBlock(11, 10) + space.AddBlock(23, 10) + fmt.Println(space.GetAllBlocks()) + fmt.Println(space.Contains(24, 10)) + //t1 := gtime.Microsecond() + //for i := 1; i <= 10; i++ { + // space.AddBlock(i*grand.Rand(0, 10000000), uint(i*10)) + // //space.AddBlock(i, uint(i*100)) + // //fmt.Println(space.GetAllBlocks()) + //} + //fmt.Println("create", gtime.Microsecond() - t1) + //e := space.Export() + //space2 := gfilespace.New() + //fmt.Println(e) + //space2.Import(e) + //fmt.Println(space2.Export()) + //t2 := gtime.Microsecond() + //fmt.Println(space.GetBlock(10)) + //fmt.Println(space.GetBlock(10)) + //fmt.Println(space.GetBlock(10)) + //fmt.Println(space.GetBlock(10)) + //fmt.Println(space.GetBlock(10)) + //fmt.Println(space.GetBlock(10)) + //fmt.Println(space.GetBlock(10)) + //fmt.Println("get", gtime.Microsecond() - t2) + // + //fmt.Println(space.GetAllBlocks()) + //fmt.Println(space.GetAllSizes()) + + + + //add block: 1792 192 + //[{0 192} {512 192} {768 384} {1408 960}] + //add block: 320 192 + //[{0 192} {320 192} {512 192} {768 384} {1408 960}] + + //add mt block 1618432 64 + //[{1618432 64}] + //[{1618432 64}] + //add mt block 1618496 64 + //[{1618432 128}] + //[{1618432 64}] + //space.AddBlock(467264, 64) + //space.AddBlock(467200, 128) + + + + + //space.Empty() + + //fmt.Println(gtime.Microsecond() - t1) + + //fmt.Println(space.GetBlock(15)) + //fmt.Println(space.GetBlock(15)) +} \ No newline at end of file diff --git a/gexample/other/reflect.go b/gexample/other/reflect.go new file mode 100644 index 000000000..e528f6e28 --- /dev/null +++ b/gexample/other/reflect.go @@ -0,0 +1,60 @@ +package main + +import ( + "fmt" + //"reflect" + "reflect" +) +//import "reflect" + +type gtInterface interface { + Run() +} + +type st struct { + age int + name string +} + +type mySt struct { + st +} + +func (_ st) Echo(str string) { + fmt.Printf("echo(%s)\n", str) +} +func (_ *st) Echo2(str string) { + fmt.Printf("echo2(%s)\n", str) +} + +func (_ st) Echo3() { + fmt.Println("echo3()") +} + +func Echo3() { + fmt.Println("echo3()") +} + +type DefaultFunc func() + +func Call(i DefaultFunc) { + i() + //reflect.ValueOf(i).Call([]reflect.Value{}) +} +func main() { + s := st {16,"john"} + + + //p := reflect.ValueOf("halloo") + v := reflect.ValueOf(s) + //v2 := reflect.ValueOf(&s) + //// 调用st结构体的方法 + //v.MethodByName("Echo").Call([]reflect.Value{p}) + //// 我们需要调用的是实体结构体指针的方法，注意v2与v2的区别，以及方法定义的区别 + //v2.MethodByName("Echo2").Call([]reflect.Value{p}) + //v.MethodByName() + fmt.Println(v.Type()) + + + +} \ No newline at end of file diff --git a/gexample/other/test.go b/gexample/other/test.go new file mode 100644 index 000000000..39d4d6855 --- /dev/null +++ b/gexample/other/test.go @@ -0,0 +1,261 @@ +package main + +import ( + "fmt" + "sync" +) + + + + +func main() { + var mu sync.RWMutex + + mu.Lock() + mu.RLock() + + fmt.Println(1) + + mu.RUnlock() + mu.Unlock() + + return + //a := 1497965 + //for i := 0; i < 10000; i++ { + // fmt.Println(a*i) + //} + // + //fmt.Println(math.MaxUint64) + //m := make(map[int]int, 0) + //for i := 0; i < 10000000; i ++ { + // m[i] = i + //} + //t1 := gtime.Microsecond() + //for i := 0; i < 10; i ++ { + // if _, ok := m[i]; ok { + // + // } + //} + + //b := make([]byte, 100000) + //removeBlock(b, 80000) + //fmt.Println(gtime.Microsecond() - t1) + + return + //slice := []int{1,2,3,4,5,6,7,8,9} + //index := 1 + ////fmt.Println(append(slice[:index], slice[index+1:]...)) + // + ////rear:=append([]int{}, slice[index:]...) + ////slice=append(slice[0:index], 88) + ////slice=append(slice, rear...) + //// + ////fmt.Println(slice) + // + //fmt.Println(append(append(slice[0 : index], 88), append([]int{}, slice[index : ]...)...)) + //return + //a := gbinary.EncodeBits(nil, 100, 10) + //fmt.Println(a) + //b := gbinary.EncodeBitsToBytes(a) + //fmt.Println(b) + //fmt.Println(gbinary.EncodeInt32(1)) + //return + //return + + //fmt.Println(gbinary.DecodeToInt64([]byte{1})) + //return + //fmt.Println(gbinary.EncodeInt32(1)[0:3]) + //b := []int{1,2,3} + //c := []int{4} + //copy(b[1:], c) + //fmt.Println(b) + //return + //space, err := gfilespace.New("/tmp/test") + //if err != nil { + // fmt.Println(err) + //} + //for i := 0; i < 10; i++ { + // space.AddBlock(int64(i), uint32((i + 1)*10)) + //} + //fmt.Println(space.GetBlock(50)) + //return + + + //db.Set([]byte("1"), []byte(grand.RandStr(10))) + //grand.RandStr(10) + //db.Set([]byte("r88U89b6Vv"), []byte("john211111111111111111111111")) + //db.Get([]byte("name2")) + //fmt.Println(e) + + //fmt.Println(string(v)) + //r := int32(binary.LittleEndian.Uint32(b)) + //fmt.Println(int32(r)) + //binary.BigEndian.Uint16(b) + //gbinary.DecodeToInt32([]byte{1,2,3,4}) + //fmt.Println(gtime.Microsecond() - t1) + //fmt.Println([]byte{byte(i)}) + + //b := make([]byte, 0) + //a := ghash.BKDRHash([]byte("john")) + //for i := 0; i < 1000; i++ { + // r, e := gbinary.Encode([]byte("key_" + strconv.Itoa(i)), a, a) + // if e != nil { + // fmt.Println(e) + // return + // } + // b = append(b, r...) + //} + //fmt.Printf("length: %d\n", len(b)/1024) + //fmt.Printf("compressed: %d\n", len(gcompress.Zlib(b))/1024) + //t1 := gtime.Microsecond() + ////gcompress.Zlib(b) + //gbinary.Encode([]byte("key_" + strconv.Itoa(100)), a, a) + //fmt.Println(gtime.Microsecond() - t1) + //return + //t1 := gtime.Second() + //m := make(map[uint64]bool) + //c := 0 + //for i := 0; i < 100000000; i++ { + // key := ghash.SDBMHash64([]byte("this is test key" + strconv.Itoa(i))) + // if _, ok := m[key]; ok { + // c++ + // } else { + // m[key] = true + // } + //} + //fmt.Println(gtime.Second() - t1) + //fmt.Println("conflicts:", c) + //fmt.Println(ghash.BKDRHash([]byte("johnWRWEREWREWRWEREWRRRRRRRRRRRRRRRRRRRRRRRRRRRRR"))) + //t1 := gtime.Microsecond() + //ghash.BKDRHash64([]byte("john")) + ////fmt.Println(ghash.ELFHash([]byte("john"))) + ////fmt.Println(ghash.JSHash([]byte("john"))) + //fmt.Println(gtime.Microsecond() - t1) + //fmt.Println(ghash.BKDRHash64([]byte("john29384723894723894789sdkjfhsjkdh"))) + //return + //btree := gbtree.New(3) + //t1 := gtime.Microsecond() + //for i := 1; i <= 11; i++ { + // btree.Set([]byte{byte(i)}, []byte{byte(i)}) + //} + //fmt.Println(gtime.Microsecond() - t1) + //btree.Print() + //fmt.Println() + //fmt.Println() + //btree.Remove([]byte{11}) + //btree.Print() + + //t2 := gtime.Microsecond() + //btree.Get([]byte("key2")) + //fmt.Println(btree.Get([]byte{200})) + //fmt.Println(gtime.Microsecond() - t2) + + //return + ////m := gmap.NewStringInterfaceMap() + //t1 := gtime.Microsecond() + //gcrc32.EncodeString("123") + //fmt.Println(gtime.Microsecond() - t1) + //return + //db, err := leveldb.OpenFile("/tmp/lv.db", nil) + //fmt.Println(err) + //defer db.Close() + //t1 := gtime.Microsecond() + //size := 10000000 + + //for i := 0; i < size; i++ { + // //r := []byte(grand.RandStr(10)) + // //if err := db.Set(r, r); err != nil { + // t3 := gtime.Microsecond() + // if err := db.Put([]byte("key1_" + strconv.Itoa(i)), []byte("value1_" + strconv.Itoa(i)), nil); err != nil { + // //if err := db.Set(gbinary.EncodeInt32(int32(i)), gbinary.EncodeInt32(int32(i))); err != nil { + // fmt.Println(err) + // } + // t4 := gtime.Microsecond() + // if t4 - t3 > 1000 { + // fmt.Println(t4-t3) + // } + //} + + //for i := 0; i < size; i++ { + // //r := []byte(grand.RandStr(10)) + // //if err := db.Set(r, r); err != nil { + // t3 := gtime.Microsecond() + // v, err := db.Get([]byte("key1_" + strconv.Itoa(i)), nil) + // if err != nil { + // //if err := db.Set(gbinary.EncodeInt32(int32(i)), gbinary.EncodeInt32(int32(i))); err != nil { + // fmt.Println(err) + // } + // if len(v) == 0 { + // fmt.Println("none") + // } + // t4 := gtime.Microsecond() + // if t4 - t3 > 1000 { + // fmt.Println(t4-t3) + // } + //} + //fmt.Println(gtime.Microsecond() - t1) + // + //return + //db, err := bolt.Open("/tmp/my.db", 0600, nil) + //if err != nil { + // log.Fatal(err) + //} + //defer db.Close() + // + //tx, err := db.Begin(true) + //if err != nil { + // log.Fatal(err) + //} + //defer tx.Rollback() + + // Use the transaction... + //_, err = tx.CreateBucket([]byte("MyBucket")) + //if err != nil { + // log.Fatal(err) + //} + + // Commit the transaction and check for error. + //if err := tx.Commit(); err != nil { + // log.Fatal(err) + //} + //t1 := gtime.Microsecond() + //db.Update(func(tx *bolt.Tx) error { + // b := tx.Bucket([]byte("MyBucket")) + // err := b.Put([]byte("answer"), []byte("11")) + // return err + //}) + //fmt.Println(gtime.Microsecond() - t1) + // + //t2 := gtime.Microsecond() + //db.View(func(tx *bolt.Tx) error { + // b := tx.Bucket([]byte("MyBucket")) + // v := b.Get([]byte("answer")) + // fmt.Printf("The answer is: %s\n", v) + // return nil + //}) + //fmt.Println(gtime.Microsecond() - t2) + + + //return + ////db, err := gkvdb.New("/tmp/test2", "t") + //fmt.Println(err) + ////fmt.Println(db.Set("1", []byte("1"))) + //t1 := gtime.Microsecond() + ////fmt.Println(db.Get("1")) + //fmt.Println(db.Set("1", []byte("1"))) + //fmt.Println(gtime.Microsecond() - t1) + ////fmt.Println(db.Set("name", []byte("222"))) + //return + //for i := 0; i < 10000000; i++ { + // gfile.PutContentsAppend("/tmp/test", "1234567890") + //} + // + //file, _ := gfile.OpenWithFlag("/tmp/test", os.O_RDWR|os.O_CREATE) + //fmt.Println(gfile.GetBinContentByTwoOffsets(file, 100, 110)) + ////n, err := file.WriteAt([]byte("123"), 2286 445 522*8) + ////n, err := file.WriteAt([]byte("123"), 1000000*(16)) + ////fmt.Println(n) + ////fmt.Println(err) + //defer file.Close() + //fmt.Println(gcrc32.EncodeString("123")) +} \ No newline at end of file diff --git a/gexample/other/test_test.go b/gexample/other/test_test.go new file mode 100644 index 000000000..a7f88b2de --- /dev/null +++ b/gexample/other/test_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "testing" + "g/core/types/gmap" +) + + + + +func BenchmarkSet1(b *testing.B) { + a := gmap.NewStringStringMap() + m1 := make(map[string]string) + m2 := make(map[string]string) + for i := 0; i < 1000000; i ++ { + m1[string(i)] = string(i) + } + for i := 0; i < 1000000; i ++ { + m2[string(i)] = string(i) + "_2" + } + a.BatchSet(m1) + a.BatchSet(m2) +} + +func BenchmarkSet2 (b *testing.B) { + a := gmap.NewStringStringMap() + m1 := make(map[string]string) + m2 := make(map[string]string) + for i := 0; i < 1000000; i ++ { + m1[string(i)] = string(i) + } + for i := 0; i < 1000000; i ++ { + m2[string(i)] = string(i) + "_2" + } + a.BatchSet2(m1) + a.BatchSet2(m2) +} \ No newline at end of file diff --git a/gexample/types/gbtree.go b/gexample/types/gbtree.go new file mode 100644 index 000000000..f1d2a82dd --- /dev/null +++ b/gexample/types/gbtree.go @@ -0,0 +1,44 @@ +package main + +import ( + "g/core/types/gbtree" + "fmt" +) + +type Block struct { + index int // 文件偏移量 + size uint // 区块大小(byte) +} + +func (block *Block) Less(item gbtree.Item) bool { + if block.index < item.(*Block).index { + return true + } + return false +} + +func main () { + tr := gbtree.New(10) + + //t1 := gtime.Microsecond() + for i := 0; i < 10; i++ { + tr.ReplaceOrInsert(&Block{i, uint(i*10)}) + } + //fmt.Println("create", gtime.Microsecond() - t1) + + //t2 := gtime.Microsecond() + //b := &Block{9, 10} + //fmt.Println(tr.Get(b)) + //fmt.Println(tr.Delete(b)) + //fmt.Println(tr.Get(b)) + //fmt.Println("get", gtime.Microsecond() - t2) + + //t3 := gtime.Microsecond() + //var b Block + tr.AscendGreaterOrEqual(&Block{2, 0}, func(item gbtree.Item) bool { + fmt.Println(item) + return true + }) + //fmt.Println("asc fetch", gtime.Microsecond() - t3, b) + +}