[PATCH RESEND] remote-curl: show progress for fetches over dumb HTTP

[PATCH RESEND] remote-curl: show progress for fetches over dumb HTTP

[René Scharfe]

Fetching over dumb HTTP transport doesn't show any progress, even with
the option --progress.  If the connection is slow or there is a lot of
data to get then this can take a long time while the user is left to
wonder if git got stuck.

We don't know the number of objects to fetch at the outset, but we can
count the ones we got.  Show an open-ended progress indicator based on
that number if the user asked for it.

Signed-off-by: René Scharfe <l.s.r@web.de>
---
Original submission:
https://lore.kernel.org/git/9ed26e7e-c19c-cdb2-0710-3b91bf31291b@web.de/

 remote-curl.c |  1 +
 walker.c      | 13 ++++++++++++-
 walker.h      |  1 +
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/remote-curl.c b/remote-curl.c
index 8eb96152f5..e4cd321844 100644
--- a/remote-curl.c
+++ b/remote-curl.c
@@ -1026,6 +1026,7 @@ static int fetch_dumb(int nr_heads, struct ref **to_fetch)

 	walker = get_http_walker(url.buf);
 	walker->get_verbosely = options.verbosity >= 3;
+	walker->get_progress = options.progress;
 	walker->get_recover = 0;
 	ret = walker_fetch(walker, nr_heads, targets, NULL, NULL);
 	walker_free(walker);
diff --git a/walker.c b/walker.c
index bb010f7a2b..4984bf8b3d 100644
--- a/walker.c
+++ b/walker.c
@@ -8,6 +8,7 @@
 #include "tag.h"
 #include "blob.h"
 #include "refs.h"
+#include "progress.h"

 static struct object_id current_commit_oid;

@@ -162,6 +163,11 @@ static int process(struct walker *walker, struct object *obj)
 static int loop(struct walker *walker)
 {
 	struct object_list *elem;
+	struct progress *progress = NULL;
+	uint64_t nr = 0;
+
+	if (walker->get_progress)
+		progress = start_delayed_progress(_("Fetching objects"), 0);

 	while (process_queue) {
 		struct object *obj = process_queue->item;
@@ -176,15 +182,20 @@ static int loop(struct walker *walker)
 		 */
 		if (! (obj->flags & TO_SCAN)) {
 			if (walker->fetch(walker, obj->oid.hash)) {
+				stop_progress(&progress);
 				report_missing(obj);
 				return -1;
 			}
 		}
 		if (!obj->type)
 			parse_object(the_repository, &obj->oid);
-		if (process_object(walker, obj))
+		if (process_object(walker, obj)) {
+			stop_progress(&progress);
 			return -1;
+		}
+		display_progress(progress, ++nr);
 	}
+	stop_progress(&progress);
 	return 0;
 }

diff --git a/walker.h b/walker.h
index 6d8ae00e5b..d40b016bab 100644
--- a/walker.h
+++ b/walker.h
@@ -10,6 +10,7 @@ struct walker {
 	int (*fetch)(struct walker *, unsigned char *sha1);
 	void (*cleanup)(struct walker *);
 	int get_verbosely;
+	int get_progress;
 	int get_recover;

 	int corrupt_object_found;
--
2.25.1

Re: [PATCH RESEND] remote-curl: show progress for fetches over dumb HTTP

[Junio C Hamano]

René Scharfe <l.s.r@web.de> writes:

> Fetching over dumb HTTP transport doesn't show any progress, even with
> the option --progress.  If the connection is slow or there is a lot of
> data to get then this can take a long time while the user is left to
> wonder if git got stuck.
>
> We don't know the number of objects to fetch at the outset, but we can
> count the ones we got.  Show an open-ended progress indicator based on
> that number if the user asked for it.
>
> Signed-off-by: René Scharfe <l.s.r@web.de>
> ---
> Original submission:
> https://lore.kernel.org/git/9ed26e7e-c19c-cdb2-0710-3b91bf31291b@web.de/

Thanks.

>
>  remote-curl.c |  1 +
>  walker.c      | 13 ++++++++++++-
>  walker.h      |  1 +
>  3 files changed, 14 insertions(+), 1 deletion(-)
>
> diff --git a/remote-curl.c b/remote-curl.c
> index 8eb96152f5..e4cd321844 100644
> --- a/remote-curl.c
> +++ b/remote-curl.c
> @@ -1026,6 +1026,7 @@ static int fetch_dumb(int nr_heads, struct ref **to_fetch)
>
>  	walker = get_http_walker(url.buf);
>  	walker->get_verbosely = options.verbosity >= 3;
> +	walker->get_progress = options.progress;
>  	walker->get_recover = 0;
>  	ret = walker_fetch(walker, nr_heads, targets, NULL, NULL);
>  	walker_free(walker);
> diff --git a/walker.c b/walker.c
> index bb010f7a2b..4984bf8b3d 100644
> --- a/walker.c
> +++ b/walker.c
> @@ -8,6 +8,7 @@
>  #include "tag.h"
>  #include "blob.h"
>  #include "refs.h"
> +#include "progress.h"
>
>  static struct object_id current_commit_oid;
>
> @@ -162,6 +163,11 @@ static int process(struct walker *walker, struct object *obj)
>  static int loop(struct walker *walker)
>  {
>  	struct object_list *elem;
> +	struct progress *progress = NULL;
> +	uint64_t nr = 0;
> +
> +	if (walker->get_progress)
> +		progress = start_delayed_progress(_("Fetching objects"), 0);
>
>  	while (process_queue) {
>  		struct object *obj = process_queue->item;
> @@ -176,15 +182,20 @@ static int loop(struct walker *walker)
>  		 */
>  		if (! (obj->flags & TO_SCAN)) {
>  			if (walker->fetch(walker, obj->oid.hash)) {
> +				stop_progress(&progress);
>  				report_missing(obj);
>  				return -1;
>  			}
>  		}
>  		if (!obj->type)
>  			parse_object(the_repository, &obj->oid);
> -		if (process_object(walker, obj))
> +		if (process_object(walker, obj)) {
> +			stop_progress(&progress);
>  			return -1;
> +		}
> +		display_progress(progress, ++nr);
>  	}
> +	stop_progress(&progress);
>  	return 0;
>  }
>
> diff --git a/walker.h b/walker.h
> index 6d8ae00e5b..d40b016bab 100644
> --- a/walker.h
> +++ b/walker.h
> @@ -10,6 +10,7 @@ struct walker {
>  	int (*fetch)(struct walker *, unsigned char *sha1);
>  	void (*cleanup)(struct walker *);
>  	int get_verbosely;
> +	int get_progress;
>  	int get_recover;
>
>  	int corrupt_object_found;
> --
> 2.25.1

Re: [PATCH RESEND] remote-curl: show progress for fetches over dumb HTTP

[Manish Devgan]

On Wed, Mar 4, 2020 at 2:25 AM René Scharfe <l.s.r@web.de> wrote:
>
> Fetching over dumb HTTP transport doesn't show any progress, even with
> the option --progress.  If the connection is slow or there is a lot of
> data to get then this can take a long time while the user is left to
> wonder if git got stuck.
>
> We don't know the number of objects to fetch at the outset, but we can
> count the ones we got.  Show an open-ended progress indicator based on
> that number if the user asked for it.
>
> Signed-off-by: René Scharfe <l.s.r@web.de>
> ---
> Original submission:
> https://lore.kernel.org/git/9ed26e7e-c19c-cdb2-0710-3b91bf31291b@web.de/
>
>  remote-curl.c |  1 +
>  walker.c      | 13 ++++++++++++-
>  walker.h      |  1 +
>  3 files changed, 14 insertions(+), 1 deletion(-)
>
> diff --git a/remote-curl.c b/remote-curl.c
> index 8eb96152f5..e4cd321844 100644
> --- a/remote-curl.c
> +++ b/remote-curl.c
> @@ -1026,6 +1026,7 @@ static int fetch_dumb(int nr_heads, struct ref **to_fetch)
>
>         walker = get_http_walker(url.buf);
>         walker->get_verbosely = options.verbosity >= 3;
> +       walker->get_progress = options.progress;
>         walker->get_recover = 0;
>         ret = walker_fetch(walker, nr_heads, targets, NULL, NULL);
>         walker_free(walker);
> diff --git a/walker.c b/walker.c
> index bb010f7a2b..4984bf8b3d 100644
> --- a/walker.c
> +++ b/walker.c
> @@ -8,6 +8,7 @@
>  #include "tag.h"
>  #include "blob.h"
>  #include "refs.h"
> +#include "progress.h"
>
>  static struct object_id current_commit_oid;
>
> @@ -162,6 +163,11 @@ static int process(struct walker *walker, struct object *obj)
>  static int loop(struct walker *walker)
>  {
>         struct object_list *elem;
> +       struct progress *progress = NULL;
> +       uint64_t nr = 0;
> +
> +       if (walker->get_progress)
> +               progress = start_delayed_progress(_("Fetching objects"), 0);
>
>         while (process_queue) {
>                 struct object *obj = process_queue->item;
> @@ -176,15 +182,20 @@ static int loop(struct walker *walker)
>                  */
>                 if (! (obj->flags & TO_SCAN)) {
>                         if (walker->fetch(walker, obj->oid.hash)) {
> +                               stop_progress(&progress);
>                                 report_missing(obj);
>                                 return -1;
>                         }
>                 }
>                 if (!obj->type)
>                         parse_object(the_repository, &obj->oid);
> -               if (process_object(walker, obj))
> +               if (process_object(walker, obj)) {
> +                       stop_progress(&progress);
>                         return -1;
> +               }
> +               display_progress(progress, ++nr);
>         }
> +       stop_progress(&progress);
>         return 0;
>  }
>
> diff --git a/walker.h b/walker.h
> index 6d8ae00e5b..d40b016bab 100644
> --- a/walker.h
> +++ b/walker.h
> @@ -10,6 +10,7 @@ struct walker {
>         int (*fetch)(struct walker *, unsigned char *sha1);
>         void (*cleanup)(struct walker *);
>         int get_verbosely;
> +       int get_progress;
>         int get_recover;
>
>         int corrupt_object_found;
> --
> 2.25.1


Thanks. This works for me.

Regards
Manish Devgan