Number of unique photos from multi-page search is too low

Topics: User Forum
Jan 4, 2012 at 2:55 PM
Edited Jan 4, 2012 at 3:12 PM

I've got problem with multi-page search.

I create my search options, pass them into the search, assign current page to +1 and expect to see pages * itemsPerPage number of unique photos. Instead unique photos are depleted in the first 20 or so of 500-items pages.

In the following code instead of 50.000 of photos in the end of search I've got 4491 unique photos according to the last run. I don't know whether the problem is in my code, in the base Flickr REST API, or in the FlickrNet Library. So I don't know if it's an issue or not. Can you help me with this problem please?

Best regards, Artem Koval.

 

using System;
using System.Collections.Generic;
using System.IO;
using FlickrNet;

namespace FlickrNetAPILibraryTest
{
    public class Program
    {
        public static void Main()
        {
            try
            {
                const string apiKey = ""; // your API key
                const string sharedSecret = ""; // your shared secret
                
                var flickr = new Flickr(apiKey, sharedSecret);

                var photoSearchOptions = new PhotoSearchOptions
                    {
                        PerPage = 500,
                        SortOrder = 
                        PhotoSearchSortOrder.InterestingnessDescending,
                        MediaType = MediaType.Photos,
                        ContentType = ContentTypeSearch.PhotosOnly,
                    };

                photoSearchOptions.Licenses.Add(LicenseType.AttributionCC);

                var uniquePhotos = new Dictionary<string, string>();
                var counter = 1;

                for (var i = 1; i <= 100; i++)
                {
                    var initialCounter = counter;

                    photoSearchOptions.Page = i;

                    var logRequestInfo = "Requesting page: " + photoSearchOptions.Page;
                    Console.WriteLine(logRequestInfo);

                    var photos = flickr.PhotosSearch(photoSearchOptions);

                    foreach (var photo in photos)
                    {
                        if (uniquePhotos.ContainsKey(photo.PhotoId))
                        {
                            continue;
                        }

                        var logPhotoInfo = 
                            "Item number " + counter + " " + "with ID " + photo.PhotoId;
                        Console.WriteLine(logPhotoInfo);

                        uniquePhotos.Add(photo.PhotoId, photo.LargeUrl);

                        counter++;
                    }

                    if (initialCounter != counter)
                    {
                        continue;
                    }

                    const string noItemsMessage = "No new unique items since last request.";

                    Console.WriteLine(noItemsMessage);
                }

                Console.WriteLine(uniquePhotos.Count);
            }
            catch (Exception exception)
            {
                Console.WriteLine(exception.Message);
            }

            Console.WriteLine("Press any key to continue...");
            Console.ReadLine();
        }
    }
}

Coordinator
Jan 4, 2012 at 3:44 PM

Flickr only returns 4000 unique photos for a particular search. If you want more than that then you'll have to add min/max upload dates and get 1000 per month for a year or something like that.

Sam

Jan 5, 2012 at 9:20 AM
Edited Jan 5, 2012 at 9:21 AM

Thanks for reply!

Following your advice I developed an algorithm for browsing photos with time frame of a month. But I've got new problem. Somehow despite the fact that I set photoSearchOptions to contain photos only with AttributionCC license I've got photos with AllRightsReserved for example. Where am I wrong?

 

using System;
using System.Collections.Generic;
using FlickrNet;

namespace FlickrNetAPILibraryTest
{
    public class Program
    {
        public static void Main()
        {
            try
            {
                const string apiKey = "";
                const string sharedSecret = "";

                var flickr = new Flickr(apiKey, sharedSecret);

                var photoSearchOptions = new PhotoSearchOptions
                                             {
                                                 PerPage = 500,
                                                 SortOrder = PhotoSearchSortOrder.InterestingnessDescending,
                                                 MediaType = MediaType.Photos,
                                                 ContentType = ContentTypeSearch.PhotosOnly,
                                                 MinUploadDate = new DateTime(2011, 12, 1),
                                                 MaxUploadDate = new DateTime(2011, 12, 31)
                                             };

                photoSearchOptions.Licenses.Add(LicenseType.AttributionCC);

                var uniquePhotos = new Dictionary<string, Photo>();
                var counter = 1;

                while (uniquePhotos.Count <= 10000)
                {
                    for (var i = 1; i <= 2; i++)
                    {
                        var initialCounter = counter;

                        photoSearchOptions.Page = i;

                        var logRequestInfo = "Requesting page " + photoSearchOptions.Page +
                            " for interval " + 
                            photoSearchOptions.MinUploadDate + " --- " + photoSearchOptions.MaxUploadDate;
                        Console.WriteLine(logRequestInfo);

                        var photos = flickr.PhotosSearch(photoSearchOptions);                        

                        if (photos.Count != 0 && photos[0].License != LicenseType.AttributionCC)
                        {
                            Console.WriteLine("No photos with AttributionCC license. Total number of photos: " + photos.Count);

                            continue;
                        }

                        foreach (var photo in photos)
                        {
                            if (uniquePhotos.ContainsKey(photo.PhotoId) || photo.License != LicenseType.AttributionCC)
                            {
                                continue;
                            }

                            var logPhotoInfo =
                                "Item number " + counter + " " + "with ID " + photo.PhotoId + 
                                " and license " + photo.License;
                            Console.WriteLine(logPhotoInfo);

                            uniquePhotos.Add(photo.PhotoId, photo);

                            counter++;
                        }

                        if (initialCounter != counter)
                        {
                            continue;
                        }

                        const string noItemsMessage = "No new unique items since last request.";

                        Console.WriteLine(noItemsMessage);
                    }

                    photoSearchOptions.MinUploadDate = photoSearchOptions.MinUploadDate.AddDays(-30);
                    photoSearchOptions.MaxUploadDate = photoSearchOptions.MaxUploadDate.AddDays(-30);
                }

                Console.WriteLine(uniquePhotos.Count);
            }
            catch (Exception exception)
            {
                Console.WriteLine(exception.Message);
            }

            Console.WriteLine("Press any key to continue...");
            Console.ReadLine();
        }
    }
}

Coordinator
Jan 5, 2012 at 9:22 AM

You're not actually returning the license, you need to set PhotoSearchOptions.Extras to include PhotoSearchExtras.License.

Sam

Jan 7, 2012 at 7:10 AM

Thanks for your help! Here is working code to do following thing: get more than 10.000 of unique images sorted by interestingness descending, with the sliding time frame of 1 month.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using FlickrNet;

namespace FlickrNetAPILibraryTest
{
    public class Program
    {
        public static void Main()
        {
            try
            {
                var stopWatch = new Stopwatch();
                stopWatch.Start();

                const string apiKey = "";
                const string sharedSecret = "";

                var flickr = new Flickr(apiKey, sharedSecret);

                var photoSearchOptions = new PhotoSearchOptions
                                             {
                                                 ContentType = ContentTypeSearch.PhotosOnly,
                                                 Extras = PhotoSearchExtras.All,
                                                 MaxUploadDate = new DateTime(2011, 12 , 31),
                                                 MediaType = MediaType.Photos,
                                                 MinUploadDate = new DateTime(2011, 12, 1),
                                                 PerPage = 500,
                                                 SafeSearch = SafetyLevel.Safe,
                                                 SortOrder = PhotoSearchSortOrder.InterestingnessDescending
                                             };

                photoSearchOptions.Licenses.Add(LicenseType.AttributionCC);

                var uniquePhotos = new Dictionary<string, Photo>();
                var counter = 1;

                while (uniquePhotos.Count <= 10000)
                {
                    for (var i = 1; i <= 10; i++)
                    {
                        var initialCounter = counter;

                        photoSearchOptions.Page = i;

                        var logRequestInfo = "Requesting page " + photoSearchOptions.Page +
                                                " for interval " +
                                                photoSearchOptions.MinUploadDate + " --- " +
                                                photoSearchOptions.MaxUploadDate;
                        Console.WriteLine(logRequestInfo);

                        var photos = flickr.PhotosSearch(photoSearchOptions);

                        foreach (var photo in photos)
                        {
                            if (uniquePhotos.ContainsKey(photo.PhotoId))
                            {
                                continue;
                            }

                            var logPhotoInfo =
                                "Item number " + counter + " " + "with ID " + photo.PhotoId +
                                " and license " + photo.License;
                            Console.WriteLine(logPhotoInfo);

                            uniquePhotos.Add(photo.PhotoId, photo);

                            counter++;
                        }

                        if (initialCounter != counter)
                        {
                            continue;
                        }

                        const string noItemsMessage = "No new unique items since last request.";

                        Console.WriteLine(noItemsMessage);
                    }

                    photoSearchOptions.MinUploadDate = photoSearchOptions.MinUploadDate.AddDays(-30);
                    photoSearchOptions.MaxUploadDate = photoSearchOptions.MaxUploadDate.AddDays(-30);
                }

                Console.WriteLine(uniquePhotos.Count);
                stopWatch.Stop();

                Console.WriteLine("Mining completed in: " + stopWatch.ElapsedMilliseconds/1000/60 + " minutes.");
            }
            catch (Exception exception)
            {
                Console.WriteLine(exception.Message);
            }

            Console.WriteLine("Press any key to continue...");
            Console.ReadLine();
        }
    }
}