1// Copyright 2016 Google Inc. All Rights Reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15// Package iterator provides support for standard Google API iterators. 16// See https://github.com/GoogleCloudPlatform/gcloud-golang/wiki/Iterator-Guidelines. 17package iterator 18 19import ( 20 "errors" 21 "fmt" 22 "reflect" 23) 24 25// Done is returned by an iterator's Next method when the iteration is 26// complete; when there are no more items to return. 27var Done = errors.New("no more items in iterator") 28 29// We don't support mixed calls to Next and NextPage because they play 30// with the paging state in incompatible ways. 31var errMixed = errors.New("iterator: Next and NextPage called on same iterator") 32 33// PageInfo contains information about an iterator's paging state. 34type PageInfo struct { 35 // Token is the token used to retrieve the next page of items from the 36 // API. You may set Token immediately after creating an iterator to 37 // begin iteration at a particular point. If Token is the empty string, 38 // the iterator will begin with the first eligible item. 39 // 40 // The result of setting Token after the first call to Next is undefined. 41 // 42 // After the underlying API method is called to retrieve a page of items, 43 // Token is set to the next-page token in the response. 44 Token string 45 46 // MaxSize is the maximum number of items returned by a call to the API. 47 // Set MaxSize as a hint to optimize the buffering behavior of the iterator. 48 // If zero, the page size is determined by the underlying service. 49 // 50 // Use Pager to retrieve a page of a specific, exact size. 51 MaxSize int 52 53 // The error state of the iterator. Manipulated by PageInfo.next and Pager. 54 // This is a latch: it starts as nil, and once set should never change. 55 err error 56 57 // If true, no more calls to fetch should be made. Set to true when fetch 58 // returns an empty page token. The iterator is Done when this is true AND 59 // the buffer is empty. 60 atEnd bool 61 62 // Function that fetches a page from the underlying service. It should pass 63 // the pageSize and pageToken arguments to the service, fill the buffer 64 // with the results from the call, and return the next-page token returned 65 // by the service. The function must not remove any existing items from the 66 // buffer. If the underlying RPC takes an int32 page size, pageSize should 67 // be silently truncated. 68 fetch func(pageSize int, pageToken string) (nextPageToken string, err error) 69 70 // Function that clears the iterator's buffer, returning any currently buffered items. 71 bufLen func() int 72 73 // Function that returns the buffer, after setting the buffer variable to nil. 74 takeBuf func() interface{} 75 76 // Set to true on first call to PageInfo.next or Pager.NextPage. Used to check 77 // for calls to both Next and NextPage with the same iterator. 78 nextCalled, nextPageCalled bool 79} 80 81// NewPageInfo exposes internals for iterator implementations. 82// It is not a stable interface. 83var NewPageInfo = newPageInfo 84 85// If an iterator can support paging, its iterator-creating method should call 86// this (via the NewPageInfo variable above). 87// 88// The fetch, bufLen and takeBuf arguments provide access to the 89// iterator's internal slice of buffered items. They behave as described in 90// PageInfo, above. 91// 92// The return value is the PageInfo.next method bound to the returned PageInfo value. 93// (Returning it avoids exporting PageInfo.next.) 94func newPageInfo(fetch func(int, string) (string, error), bufLen func() int, takeBuf func() interface{}) (*PageInfo, func() error) { 95 pi := &PageInfo{ 96 fetch: fetch, 97 bufLen: bufLen, 98 takeBuf: takeBuf, 99 } 100 return pi, pi.next 101} 102 103// Remaining returns the number of items available before the iterator makes another API call. 104func (pi *PageInfo) Remaining() int { return pi.bufLen() } 105 106// next provides support for an iterator's Next function. An iterator's Next 107// should return the error returned by next if non-nil; else it can assume 108// there is at least one item in its buffer, and it should return that item and 109// remove it from the buffer. 110func (pi *PageInfo) next() error { 111 pi.nextCalled = true 112 if pi.err != nil { // Once we get an error, always return it. 113 // TODO(jba): fix so users can retry on transient errors? Probably not worth it. 114 return pi.err 115 } 116 if pi.nextPageCalled { 117 pi.err = errMixed 118 return pi.err 119 } 120 // Loop until we get some items or reach the end. 121 for pi.bufLen() == 0 && !pi.atEnd { 122 if err := pi.fill(pi.MaxSize); err != nil { 123 pi.err = err 124 return pi.err 125 } 126 if pi.Token == "" { 127 pi.atEnd = true 128 } 129 } 130 // Either the buffer is non-empty or pi.atEnd is true (or both). 131 if pi.bufLen() == 0 { 132 // The buffer is empty and pi.atEnd is true, i.e. the service has no 133 // more items. 134 pi.err = Done 135 } 136 return pi.err 137} 138 139// Call the service to fill the buffer, using size and pi.Token. Set pi.Token to the 140// next-page token returned by the call. 141// If fill returns a non-nil error, the buffer will be empty. 142func (pi *PageInfo) fill(size int) error { 143 tok, err := pi.fetch(size, pi.Token) 144 if err != nil { 145 pi.takeBuf() // clear the buffer 146 return err 147 } 148 pi.Token = tok 149 return nil 150} 151 152// Pageable is implemented by iterators that support paging. 153type Pageable interface { 154 // PageInfo returns paging information associated with the iterator. 155 PageInfo() *PageInfo 156} 157 158// Pager supports retrieving iterator items a page at a time. 159type Pager struct { 160 pageInfo *PageInfo 161 pageSize int 162} 163 164// NewPager returns a pager that uses iter. Calls to its NextPage method will 165// obtain exactly pageSize items, unless fewer remain. The pageToken argument 166// indicates where to start the iteration. Pass the empty string to start at 167// the beginning, or pass a token retrieved from a call to Pager.NextPage. 168// 169// If you use an iterator with a Pager, you must not call Next on the iterator. 170func NewPager(iter Pageable, pageSize int, pageToken string) *Pager { 171 p := &Pager{ 172 pageInfo: iter.PageInfo(), 173 pageSize: pageSize, 174 } 175 p.pageInfo.Token = pageToken 176 if pageSize <= 0 { 177 p.pageInfo.err = errors.New("iterator: page size must be positive") 178 } 179 return p 180} 181 182// NextPage retrieves a sequence of items from the iterator and appends them 183// to slicep, which must be a pointer to a slice of the iterator's item type. 184// Exactly p.pageSize items will be appended, unless fewer remain. 185// 186// The first return value is the page token to use for the next page of items. 187// If empty, there are no more pages. Aside from checking for the end of the 188// iteration, the returned page token is only needed if the iteration is to be 189// resumed a later time, in another context (possibly another process). 190// 191// The second return value is non-nil if an error occurred. It will never be 192// the special iterator sentinel value Done. To recognize the end of the 193// iteration, compare nextPageToken to the empty string. 194// 195// It is possible for NextPage to return a single zero-length page along with 196// an empty page token when there are no more items in the iteration. 197func (p *Pager) NextPage(slicep interface{}) (nextPageToken string, err error) { 198 p.pageInfo.nextPageCalled = true 199 if p.pageInfo.err != nil { 200 return "", p.pageInfo.err 201 } 202 if p.pageInfo.nextCalled { 203 p.pageInfo.err = errMixed 204 return "", p.pageInfo.err 205 } 206 if p.pageInfo.bufLen() > 0 { 207 return "", errors.New("must call NextPage with an empty buffer") 208 } 209 // The buffer must be empty here, so takeBuf is a no-op. We call it just to get 210 // the buffer's type. 211 wantSliceType := reflect.PtrTo(reflect.ValueOf(p.pageInfo.takeBuf()).Type()) 212 if slicep == nil { 213 return "", errors.New("nil passed to Pager.NextPage") 214 } 215 vslicep := reflect.ValueOf(slicep) 216 if vslicep.Type() != wantSliceType { 217 return "", fmt.Errorf("slicep should be of type %s, got %T", wantSliceType, slicep) 218 } 219 for p.pageInfo.bufLen() < p.pageSize { 220 if err := p.pageInfo.fill(p.pageSize - p.pageInfo.bufLen()); err != nil { 221 p.pageInfo.err = err 222 return "", p.pageInfo.err 223 } 224 if p.pageInfo.Token == "" { 225 break 226 } 227 } 228 e := vslicep.Elem() 229 e.Set(reflect.AppendSlice(e, reflect.ValueOf(p.pageInfo.takeBuf()))) 230 return p.pageInfo.Token, nil 231} 232